
'''simple docstring'''
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


lowercase__		    =	logging.getLogger(__name__)

require_version("pytorch_lightning>=1.0.4")

lowercase__		    =	{
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
lowercase__		    =	{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
lowercase__		    =	sorted(arg_to_scheduler.keys())
lowercase__		    =	"{" + ", ".join(arg_to_scheduler_choices) + "}"






class      snake_case__    (	pl.LightningModule ):







		"""simple docstring"""

		def __init__(	self							:       Any   ,     UpperCamelCase__							:       argparse.Namespace   ,     UpperCamelCase__							:       Union[str, Any]=None   ,     UpperCamelCase__							:       Optional[int]="base"   ,     UpperCamelCase__							:       List[Any]=None   ,     UpperCamelCase__							:       Any=None   ,     UpperCamelCase__							:       Union[str, Any]=None   ,     **UpperCamelCase__							:       Dict   ,     )		->       Dict:






				"""simple docstring"""







				super().__init__()
				# TODO: move to self.save_hyperparameters()
				# self.save_hyperparameters()
				# can also expand arguments into trainer signature for easier reading

				self.save_hyperparameters(UpperCamelCase__	)
				snake_case   :    Any			       =		0
				snake_case   :    List[Any]			       =		Path(self.hparams.output_dir	)
				snake_case   :    Union[str, Any]			       =		self.hparams.cache_dir if self.hparams.cache_dir else None
				if config is None:
						snake_case   :    List[str]			       =		AutoConfig.from_pretrained(
						    self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path   ,     **({'''num_labels''': num_labels} if num_labels is not None else {})   ,     cache_dir=UpperCamelCase__   ,     **UpperCamelCase__   ,     )
				else:
						snake_case   :    PretrainedConfig			       =		config

				snake_case   :    Tuple			       =		('''encoder_layerdrop''', '''decoder_layerdrop''', '''dropout''', '''attention_dropout''')
				for p in extra_model_params:
						if getattr(self.hparams   ,     UpperCamelCase__   ,     UpperCamelCase__	):
								assert hasattr(self.config   ,     UpperCamelCase__	), f'model config doesn\'t have a `{p}` attribute'
								setattr(self.config   ,     UpperCamelCase__   ,     getattr(self.hparams   ,     UpperCamelCase__	)	)

				if tokenizer is None:
						snake_case   :    Any			       =		AutoTokenizer.from_pretrained(
						    self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path   ,     cache_dir=UpperCamelCase__   ,     )
				else:
						snake_case   :    PreTrainedTokenizer			       =		tokenizer
				snake_case   :    Union[str, Any]			       =		MODEL_MODES[mode]
				if model is None:
						snake_case   :    Union[str, Any]			       =		self.model_type.from_pretrained(
						    self.hparams.model_name_or_path   ,     from_tf=bool('''.ckpt''' in self.hparams.model_name_or_path	)   ,     config=self.config   ,     cache_dir=UpperCamelCase__   ,     )
				else:
						snake_case   :    int			       =		model



		def 	lowerCAmelCase							(	self							:       List[Any]   ,     *UpperCamelCase__							:       Union[str, Any]   ,     **UpperCamelCase__							:       List[Any]	)		->       Optional[int]:






				"""simple docstring"""







				snake_case   :    int			       =		self.model_type.from_pretrained(*UpperCamelCase__   ,     **UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       Optional[Any]	)		->       Tuple:






				"""simple docstring"""







				snake_case   :    List[Any]			       =		arg_to_scheduler[self.hparams.lr_scheduler]
				snake_case   :    Optional[int]			       =		get_schedule_func(
				    self.opt   ,     num_warmup_steps=self.hparams.warmup_steps   ,     num_training_steps=self.total_steps()	)
				snake_case   :    str			       =		{'''scheduler''': scheduler, '''interval''': '''step''', '''frequency''': 1}
				return scheduler



		def 	lowerCAmelCase							(	self							:       Optional[Any]	)		->       Tuple:






				"""simple docstring"""







				snake_case   :    Dict			       =		self.model
				snake_case   :    Optional[Any]			       =		['''bias''', '''LayerNorm.weight''']
				snake_case   :    Optional[int]			       =		[
				    {
				        '''params''': [p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay	)],
				        '''weight_decay''': self.hparams.weight_decay,
				    },
				    {
				        '''params''': [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay	)],
				        '''weight_decay''': 0.0,
				    },
				]
				if self.hparams.adafactor:
						snake_case   :    List[str]			       =		Adafactor(
						    UpperCamelCase__   ,     lr=self.hparams.learning_rate   ,     scale_parameter=UpperCamelCase__   ,     relative_step=UpperCamelCase__	)

				else:
						snake_case   :    Optional[Any]			       =		AdamW(
						    UpperCamelCase__   ,     lr=self.hparams.learning_rate   ,     eps=self.hparams.adam_epsilon	)
				snake_case   :    Optional[int]			       =		optimizer

				snake_case   :    Tuple			       =		self.get_lr_scheduler()

				return [optimizer], [scheduler]



		def 	lowerCAmelCase							(	self							:       Dict   ,     UpperCamelCase__							:       str   ,     UpperCamelCase__							:       Optional[int]	)		->       Dict:






				"""simple docstring"""







				return self.validation_step(UpperCamelCase__   ,     UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       List[Any]   ,     UpperCamelCase__							:       List[str]	)		->       str:






				"""simple docstring"""







				return self.validation_end(UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       Union[str, Any]	)		->       int:






				"""simple docstring"""







				snake_case   :    Union[str, Any]			       =		max(1   ,     self.hparams.gpus	)  # TODO: consider num_tpu_cores
				snake_case   :    List[Any]			       =		self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
				return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs



		def 	lowerCAmelCase							(	self							:       Dict   ,     UpperCamelCase__							:       int	)		->       str:






				"""simple docstring"""







				if mode == "test":
						snake_case   :    Any			       =		len(self.test_dataloader().dataset	)
				else:
						snake_case   :    List[str]			       =		self.get_dataloader('''train'''   ,     self.hparams.train_batch_size   ,     shuffle=UpperCamelCase__	)
						snake_case   :    List[Any]			       =		len(self.train_dataloader().dataset	)



		def 	lowerCAmelCase							(	self							:       str   ,     UpperCamelCase__							:       str   ,     UpperCamelCase__							:       int   ,     UpperCamelCase__							:       bool = False	)		->       List[Any]:






				"""simple docstring"""







				raise NotImplementedError('''You must implement this for your task'''	)



		def 	lowerCAmelCase							(	self							:       Union[str, Any]	)		->       List[str]:






				"""simple docstring"""







				return self.train_loader



		def 	lowerCAmelCase							(	self							:       List[str]	)		->       Union[str, Any]:






				"""simple docstring"""







				return self.get_dataloader('''dev'''   ,     self.hparams.eval_batch_size   ,     shuffle=UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       List[str]	)		->       List[Any]:






				"""simple docstring"""







				return self.get_dataloader('''test'''   ,     self.hparams.eval_batch_size   ,     shuffle=UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       Optional[Any]   ,     UpperCamelCase__							:       Any	)		->       Union[str, Any]:






				"""simple docstring"""







				return os.path.join(
				    self.hparams.data_dir   ,     '''cached_{}_{}_{}'''.format(
				        UpperCamelCase__   ,     list(filter(UpperCamelCase__   ,     self.hparams.model_name_or_path.split('''/'''	)	)	).pop()   ,     str(self.hparams.max_seq_length	)   ,     )   ,     )



		@pl.utilities.rank_zero_only
		def 	lowerCAmelCase							(	self							:       Optional[int]   ,     UpperCamelCase__							:       Dict[str, Any]	)		->       None:






				"""simple docstring"""







				snake_case   :    Union[str, Any]			       =		self.output_dir.joinpath('''best_tfmr'''	)
				snake_case   :    Dict			       =		self.step_count
				self.model.save_pretrained(UpperCamelCase__	)
				self.tokenizer.save_pretrained(UpperCamelCase__	)



		@staticmethod
		def 	lowerCAmelCase							(	UpperCamelCase__							:       Dict   ,     UpperCamelCase__							:       List[Any]	)		->       str:






				"""simple docstring"""







				parser.add_argument(
				    '''--model_name_or_path'''   ,     default=UpperCamelCase__   ,     type=UpperCamelCase__   ,     required=UpperCamelCase__   ,     help='''Path to pretrained model or model identifier from huggingface.co/models'''   ,     )
				parser.add_argument(
				    '''--config_name'''   ,     default=''''''   ,     type=UpperCamelCase__   ,     help='''Pretrained config name or path if not the same as model_name'''	)
				parser.add_argument(
				    '''--tokenizer_name'''   ,     default=UpperCamelCase__   ,     type=UpperCamelCase__   ,     help='''Pretrained tokenizer name or path if not the same as model_name'''   ,     )
				parser.add_argument(
				    '''--cache_dir'''   ,     default=''''''   ,     type=UpperCamelCase__   ,     help='''Where do you want to store the pre-trained models downloaded from huggingface.co'''   ,     )
				parser.add_argument(
				    '''--encoder_layerdrop'''   ,     type=UpperCamelCase__   ,     help='''Encoder layer dropout probability (Optional). Goes into model.config'''   ,     )
				parser.add_argument(
				    '''--decoder_layerdrop'''   ,     type=UpperCamelCase__   ,     help='''Decoder layer dropout probability (Optional). Goes into model.config'''   ,     )
				parser.add_argument(
				    '''--dropout'''   ,     type=UpperCamelCase__   ,     help='''Dropout probability (Optional). Goes into model.config'''   ,     )
				parser.add_argument(
				    '''--attention_dropout'''   ,     type=UpperCamelCase__   ,     help='''Attention dropout probability (Optional). Goes into model.config'''   ,     )
				parser.add_argument('''--learning_rate'''   ,     default=5e-5   ,     type=UpperCamelCase__   ,     help='''The initial learning rate for Adam.'''	)
				parser.add_argument(
				    '''--lr_scheduler'''   ,     default='''linear'''   ,     choices=UpperCamelCase__   ,     metavar=UpperCamelCase__   ,     type=UpperCamelCase__   ,     help='''Learning rate scheduler'''   ,     )
				parser.add_argument('''--weight_decay'''   ,     default=0.0   ,     type=UpperCamelCase__   ,     help='''Weight decay if we apply some.'''	)
				parser.add_argument('''--adam_epsilon'''   ,     default=1e-8   ,     type=UpperCamelCase__   ,     help='''Epsilon for Adam optimizer.'''	)
				parser.add_argument('''--warmup_steps'''   ,     default=0   ,     type=UpperCamelCase__   ,     help='''Linear warmup over warmup_steps.'''	)
				parser.add_argument('''--num_workers'''   ,     default=4   ,     type=UpperCamelCase__   ,     help='''kwarg passed to DataLoader'''	)
				parser.add_argument('''--num_train_epochs'''   ,     dest='''max_epochs'''   ,     default=3   ,     type=UpperCamelCase__	)
				parser.add_argument('''--train_batch_size'''   ,     default=32   ,     type=UpperCamelCase__	)
				parser.add_argument('''--eval_batch_size'''   ,     default=32   ,     type=UpperCamelCase__	)
				parser.add_argument('''--adafactor'''   ,     action='''store_true'''	)







class      snake_case__    (	pl.Callback ):







		"""simple docstring"""

		def 	lowerCAmelCase							(	self							:       Any   ,     UpperCamelCase__							:       Optional[Any]   ,     UpperCamelCase__							:       List[Any]	)		->       Dict:






				"""simple docstring"""







				snake_case   :    Tuple			       =		trainer.lr_schedulers[0]['''scheduler''']
				snake_case   :    Tuple			       =		{f'lr_group_{i}': lr for i, lr in enumerate(lr_scheduler.get_lr()	)}
				pl_module.logger.log_metrics(UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       List[str]   ,     UpperCamelCase__							:       pl.Trainer   ,     UpperCamelCase__							:       pl.LightningModule	)		->       Optional[int]:






				"""simple docstring"""







				rank_zero_info('''***** Validation results *****'''	)
				snake_case   :    List[Any]			       =		trainer.callback_metrics
				# Log results
				for key in sorted(UpperCamelCase__	):
						if key not in ["log", "progress_bar"]:
								rank_zero_info('''{} = {}\n'''.format(UpperCamelCase__   ,     str(metrics[key]	)	)	)



		def 	lowerCAmelCase							(	self							:       str   ,     UpperCamelCase__							:       pl.Trainer   ,     UpperCamelCase__							:       pl.LightningModule	)		->       Optional[int]:






				"""simple docstring"""







				rank_zero_info('''***** Test results *****'''	)
				snake_case   :    Dict			       =		trainer.callback_metrics
				# Log and save results to file
				snake_case   :    Tuple			       =		os.path.join(pl_module.hparams.output_dir   ,     '''test_results.txt'''	)
				with open(UpperCamelCase__   ,     '''w'''	) as writer:
						for key in sorted(UpperCamelCase__	):
								if key not in ["log", "progress_bar"]:
										rank_zero_info('''{} = {}\n'''.format(UpperCamelCase__   ,     str(metrics[key]	)	)	)
										writer.write('''{} = {}\n'''.format(UpperCamelCase__   ,     str(metrics[key]	)	)	)







def 					_UpperCamelCase		(    SCREAMING_SNAKE_CASE__      ,    SCREAMING_SNAKE_CASE__       )   ->				None:


		'''simple docstring'''


		parser.add_argument(
		    '''--output_dir'''      ,    default=SCREAMING_SNAKE_CASE__      ,    type=SCREAMING_SNAKE_CASE__      ,    required=SCREAMING_SNAKE_CASE__      ,    help='''The output directory where the model predictions and checkpoints will be written.'''      ,    )
		parser.add_argument(
		    '''--fp16'''      ,    action='''store_true'''      ,    help='''Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit'''      ,    )

		parser.add_argument(
		    '''--fp16_opt_level'''      ,    type=SCREAMING_SNAKE_CASE__      ,    default='''O2'''      ,    help=(
		        '''For fp16: Apex AMP optimization level selected in [\'O0\', \'O1\', \'O2\', and \'O3\'].'''
		        '''See details at https://nvidia.github.io/apex/amp.html'''
		    )      ,    )
		parser.add_argument('''--n_tpu_cores'''      ,    dest='''tpu_cores'''      ,    type=SCREAMING_SNAKE_CASE__       )
		parser.add_argument('''--max_grad_norm'''      ,    dest='''gradient_clip_val'''      ,    default=1.0      ,    type=SCREAMING_SNAKE_CASE__      ,    help='''Max gradient norm'''       )
		parser.add_argument('''--do_train'''      ,    action='''store_true'''      ,    help='''Whether to run training.'''       )
		parser.add_argument('''--do_predict'''      ,    action='''store_true'''      ,    help='''Whether to run predictions on the test set.'''       )
		parser.add_argument(
		    '''--gradient_accumulation_steps'''      ,    dest='''accumulate_grad_batches'''      ,    type=SCREAMING_SNAKE_CASE__      ,    default=1      ,    help='''Number of updates steps to accumulate before performing a backward/update pass.'''      ,    )
		parser.add_argument('''--seed'''      ,    type=SCREAMING_SNAKE_CASE__      ,    default=42      ,    help='''random seed for initialization'''       )
		parser.add_argument(
		    '''--data_dir'''      ,    default=SCREAMING_SNAKE_CASE__      ,    type=SCREAMING_SNAKE_CASE__      ,    required=SCREAMING_SNAKE_CASE__      ,    help='''The input data dir. Should contain the training files for the CoNLL-2003 NER task.'''      ,    )




def 					_UpperCamelCase		(    SCREAMING_SNAKE_CASE__      ,    SCREAMING_SNAKE_CASE__      ,    SCREAMING_SNAKE_CASE__=None      ,    SCREAMING_SNAKE_CASE__=True      ,    SCREAMING_SNAKE_CASE__=[]      ,    SCREAMING_SNAKE_CASE__=None      ,    SCREAMING_SNAKE_CASE__=None      ,    **SCREAMING_SNAKE_CASE__      ,    )   ->				Optional[Any]:


		'''simple docstring'''


		pl.seed_everything(args.seed       )

		# init model
		snake_case   :    Dict			       =		Path(model.hparams.output_dir       )
		odir.mkdir(exist_ok=SCREAMING_SNAKE_CASE__       )

		# add custom checkpoints
		if checkpoint_callback is None:
				snake_case   :    List[Any]			       =		pl.callbacks.ModelCheckpoint(
				    filepath=args.output_dir      ,    prefix='''checkpoint'''      ,    monitor='''val_loss'''      ,    mode='''min'''      ,    save_top_k=1       )
		if early_stopping_callback:
				extra_callbacks.append(SCREAMING_SNAKE_CASE__       )
		if logging_callback is None:
				snake_case   :    str			       =		LoggingCallback()

		snake_case   :    Union[str, Any]			       =		{}

		# TODO: remove with PyTorch 1.6 since pl uses native amp
		if args.fpaa:
				snake_case   :    List[Any]			       =		16
				snake_case   :    str			       =		args.fpaa_opt_level

		if args.gpus > 1:
				snake_case   :    str			       =		'''ddp'''

		snake_case   :    Optional[Any]			       =		args.accumulate_grad_batches
		snake_case   :    Optional[Any]			       =		extra_train_kwargs.get('''accelerator'''      ,    SCREAMING_SNAKE_CASE__       )
		snake_case   :    List[Any]			       =		extra_train_kwargs.get('''profiler'''      ,    SCREAMING_SNAKE_CASE__       )

		snake_case   :    List[str]			       =		pl.Trainer.from_argparse_args(
		    SCREAMING_SNAKE_CASE__      ,    weights_summary=SCREAMING_SNAKE_CASE__      ,    callbacks=[logging_callback] + extra_callbacks      ,    logger=SCREAMING_SNAKE_CASE__      ,    checkpoint_callback=SCREAMING_SNAKE_CASE__      ,    **SCREAMING_SNAKE_CASE__      ,    )

		if args.do_train:
				trainer.fit(SCREAMING_SNAKE_CASE__       )

		return trainer




'''simple docstring'''
import os
import tempfile
import unittest
from typing import List

from transformers.models.esm.tokenization_esm import VOCAB_FILES_NAMES, EsmTokenizer
from transformers.testing_utils import require_tokenizers
from transformers.tokenization_utils import PreTrainedTokenizer
from transformers.tokenization_utils_base import PreTrainedTokenizerBase






@require_tokenizers
class      snake_case__    (	unittest.TestCase ):







		"""simple docstring"""

		lowerCamelCase							=		EsmTokenizer



		def 	lowerCAmelCase							(	self							:       List[Any]	)		->       Any:






				"""simple docstring"""







				super().setUp()
				snake_case   :    Optional[int]			       =		tempfile.mkdtemp()
				# fmt: off
				snake_case   :    List[str]			       =		["<cls>", "<pad>", "<eos>", "<unk>", "L", "A", "G", "V", "S", "E", "R", "T", "I", "D", "P", "K", "Q", "N", "F", "Y", "M", "H", "W", "C", "X", "B", "U", "Z", "O", ".", "-", "<null_1>", "<mask>"]  # noqa: E501
				# fmt: on
				snake_case   :    List[Any]			       =		os.path.join(self.tmpdirname   ,     VOCAB_FILES_NAMES['''vocab_file''']	)
				with open(self.vocab_file   ,     '''w'''   ,     encoding='''utf-8'''	) as vocab_writer:
						vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]	)	)



		def 	lowerCAmelCase							(	self							:       str   ,     **UpperCamelCase__							:       Union[str, Any]	)		->       List[PreTrainedTokenizerBase]:






				"""simple docstring"""







				return [self.get_tokenizer(**UpperCamelCase__	)]



		def 	lowerCAmelCase							(	self							:       Union[str, Any]   ,     **UpperCamelCase__							:       Tuple	)		->       PreTrainedTokenizer:






				"""simple docstring"""







				return self.tokenizer_class.from_pretrained(self.tmpdirname   ,     **UpperCamelCase__	)



		def 	lowerCAmelCase							(	self							:       List[str]	)		->       Tuple:






				"""simple docstring"""







				snake_case   :    str			       =		self.tokenizer_class(self.vocab_file	)

				snake_case   :    Optional[int]			       =		tokenizer.tokenize('''LAGVS'''	)
				self.assertListEqual(UpperCamelCase__   ,     ['''L''', '''A''', '''G''', '''V''', '''S''']	)
				self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCamelCase__	)   ,     [4, 5, 6, 7, 8]	)



		def 	lowerCAmelCase							(	self							:       List[Any]	)		->       int:






				"""simple docstring"""







				snake_case   :    Dict			       =		self.tokenizer_class(self.vocab_file	)

				snake_case   :    List[Any]			       =		'''LAGVS'''
				self.assertListEqual(tokenizer.encode(UpperCamelCase__	)   ,     [0, 4, 5, 6, 7, 8, 2]	)



		def 	lowerCAmelCase							(	self							:       List[str]	)		->       int:






				"""simple docstring"""







				snake_case   :    Union[str, Any]			       =		self.tokenizer_class(self.vocab_file	)

				snake_case   :    int			       =		['''LAGVS''', '''WCB''']
				snake_case   :    List[str]			       =		tokenizer(UpperCamelCase__   ,     padding=UpperCamelCase__	)['''input_ids''']

				self.assertListEqual(UpperCamelCase__   ,     [[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2]]	)



		def 	lowerCAmelCase							(	self							:       int	)		->       List[str]:






				"""simple docstring"""







				snake_case   :    Dict			       =		self.tokenizer_class(self.vocab_file	)

				snake_case   :    List[str]			       =		['''LAGVS''', '''WCB''']
				snake_case   :    Any			       =		tokenizer(UpperCamelCase__   ,     padding=UpperCamelCase__	)['''input_ids''']

				self.assertListEqual(UpperCamelCase__   ,     [[0, 4, 5, 6, 7, 8, 2], [0, 22, 23, 25, 2, 1, 1]]	)



		def 	lowerCAmelCase							(	self							:       Any	)		->       int:






				"""simple docstring"""







				snake_case   :    List[str]			       =		self.get_tokenizers(fast=UpperCamelCase__	)
				for tokenizer in tokenizers:
						with self.subTest(f'{tokenizer.__class__.__name__}'	):
								snake_case   :    int			       =		'''<unk>'''
								snake_case   :    int			       =		'''<mask>'''

								snake_case   :    Any			       =		tokenizer.tokenize(UpperCamelCase__	)
								snake_case   :    Any			       =		tokenizer.tokenize(UpperCamelCase__	)

								self.assertEqual(len(UpperCamelCase__	)   ,     1	)
								self.assertEqual(len(UpperCamelCase__	)   ,     1	)
								self.assertEqual(token_a[0]   ,     UpperCamelCase__	)
								self.assertEqual(token_a[0]   ,     UpperCamelCase__	)


