



"""simple docstring"""
import argparse
import logging
import os
from pathlib import Path
from typing import Any, Dict

import pytorch_lightning as pl
from pytorch_lightning.utilities import rank_zero_info

from transformers import (
    AdamW,
    AutoConfig,
    AutoModel,
    AutoModelForPreTraining,
    AutoModelForQuestionAnswering,
    AutoModelForSeqaSeqLM,
    AutoModelForSequenceClassification,
    AutoModelForTokenClassification,
    AutoModelWithLMHead,
    AutoTokenizer,
    PretrainedConfig,
    PreTrainedTokenizer,
)
from transformers.optimization import (
    Adafactor,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)
from transformers.utils.versions import require_version


A:     List[str]										=	logging.getLogger(__name__)

require_version("pytorch_lightning>=1.0.4")

A:     Optional[int]										=	{
    "base": AutoModel,
    "sequence-classification": AutoModelForSequenceClassification,
    "question-answering": AutoModelForQuestionAnswering,
    "pretraining": AutoModelForPreTraining,
    "token-classification": AutoModelForTokenClassification,
    "language-modeling": AutoModelWithLMHead,
    "summarization": AutoModelForSeqaSeqLM,
    "translation": AutoModelForSeqaSeqLM,
}


# update this and the import above to support new schedulers from transformers.optimization
A:     Tuple										=	{
    "linear": get_linear_schedule_with_warmup,
    "cosine": get_cosine_schedule_with_warmup,
    "cosine_w_restarts": get_cosine_with_hard_restarts_schedule_with_warmup,
    "polynomial": get_polynomial_decay_schedule_with_warmup,
    # '': get_constant_schedule,             # not supported for now
    # '': get_constant_schedule_with_warmup, # not supported for now
}
A:     str										=	sorted(arg_to_scheduler.keys())
A:     Optional[Any]										=	"{" + ", ".join(arg_to_scheduler_choices) + "}"



class    SCREAMING_SNAKE_CASE__						(      pl.LightningModule				):

   def __init__(     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE="base"   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=None   ,  _SCREAMING_SNAKE_CASE=None   ,  **_SCREAMING_SNAKE_CASE   ,  )						->   int:





    '''simple docstring'''




    super().__init__()
    # TODO: move to self.save_hyperparameters()
    # self.save_hyperparameters()
    # can also expand arguments into trainer signature for easier reading

    self.save_hyperparameters(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      str			=			0
    UpperCAmelCase			:      Optional[Any]			=			Path(self.hparams.output_dir		)
    UpperCAmelCase			:      List[Any]			=			self.hparams.cache_dir if self.hparams.cache_dir else None
    if config is None:
     UpperCAmelCase			:      Optional[Any]			=			AutoConfig.from_pretrained(
         self.hparams.config_name if self.hparams.config_name else self.hparams.model_name_or_path   ,  **({"""num_labels""": num_labels} if num_labels is not None else {})   ,  cache_dir=_SCREAMING_SNAKE_CASE   ,  **_SCREAMING_SNAKE_CASE   ,  )
    else:
     UpperCAmelCase			:      PretrainedConfig			=			config

    UpperCAmelCase			:      str			=			("""encoder_layerdrop""", """decoder_layerdrop""", """dropout""", """attention_dropout""")
    for p in extra_model_params:
     if getattr(self.hparams   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		):
      assert hasattr(self.config   ,  _SCREAMING_SNAKE_CASE		), F"model config doesn't have a `{p}` attribute"
      setattr(self.config   ,  _SCREAMING_SNAKE_CASE   ,  getattr(self.hparams   ,  _SCREAMING_SNAKE_CASE		)		)

    if tokenizer is None:
     UpperCAmelCase			:      Union[str, Any]			=			AutoTokenizer.from_pretrained(
         self.hparams.tokenizer_name if self.hparams.tokenizer_name else self.hparams.model_name_or_path   ,  cache_dir=_SCREAMING_SNAKE_CASE   ,  )
    else:
     UpperCAmelCase			:      PreTrainedTokenizer			=			tokenizer
    UpperCAmelCase			:      Optional[int]			=			MODEL_MODES[mode]
    if model is None:
     UpperCAmelCase			:      Union[str, Any]			=			self.model_type.from_pretrained(
         self.hparams.model_name_or_path   ,  from_tf=bool(""".ckpt""" in self.hparams.model_name_or_path		)   ,  config=self.config   ,  cache_dir=_SCREAMING_SNAKE_CASE   ,  )
    else:
     UpperCAmelCase			:      Optional[Any]			=			model

   def 			SCREAMING_SNAKE_CASE  (     self   ,  *_SCREAMING_SNAKE_CASE   ,  **_SCREAMING_SNAKE_CASE		)						->   str:





    '''simple docstring'''




    UpperCAmelCase			:      int			=			self.model_type.from_pretrained(*_SCREAMING_SNAKE_CASE   ,  **_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Tuple			=			arg_to_scheduler[self.hparams.lr_scheduler]
    UpperCAmelCase			:      str			=			get_schedule_func(
        self.opt   ,  num_warmup_steps=self.hparams.warmup_steps   ,  num_training_steps=self.total_steps()		)
    UpperCAmelCase			:      Any			=			{"""scheduler""": scheduler, """interval""": """step""", """frequency""": 1}
    return scheduler

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    UpperCAmelCase			:      Tuple			=			self.model
    UpperCAmelCase			:      List[str]			=			["""bias""", """LayerNorm.weight"""]
    UpperCAmelCase			:      int			=			[
        {
            """params""": [
                p for n, p in model.named_parameters() if not any(nd in n for nd in no_decay		)
            ],  # check this named paramters
            """weight_decay""": self.hparams.weight_decay,
        },
        {
            """params""": [p for n, p in model.named_parameters() if any(nd in n for nd in no_decay		)],
            """weight_decay""": 0.0,
        },
    ]
    if self.hparams.adafactor:
     UpperCAmelCase			:      str			=			Adafactor(
         _SCREAMING_SNAKE_CASE   ,  lr=self.hparams.learning_rate   ,  scale_parameter=_SCREAMING_SNAKE_CASE   ,  relative_step=_SCREAMING_SNAKE_CASE		)

    else:
     UpperCAmelCase			:      Optional[int]			=			AdamW(
         _SCREAMING_SNAKE_CASE   ,  lr=self.hparams.learning_rate   ,  eps=self.hparams.adam_epsilon		)
    UpperCAmelCase			:      List[str]			=			optimizer

    UpperCAmelCase			:      str			=			self.get_lr_scheduler()

    return [optimizer], [scheduler]

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   List[Any]:





    '''simple docstring'''




    return self.validation_step(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   Dict:





    '''simple docstring'''




    return self.validation_end(_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   int:





    '''simple docstring'''




    UpperCAmelCase			:      str			=			max(1   ,  self.hparams.gpus		)  # TODO: consider num_tpu_cores
    UpperCAmelCase			:      Dict			=			self.hparams.train_batch_size * self.hparams.accumulate_grad_batches * num_devices
    return (self.dataset_size / effective_batch_size) * self.hparams.max_epochs

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   List[Any]:





    '''simple docstring'''




    if stage == "test":
     UpperCAmelCase			:      Union[str, Any]			=			len(self.test_dataloader().dataset		)
    else:
     UpperCAmelCase			:      Union[str, Any]			=			self.get_dataloader("""train"""   ,  self.hparams.train_batch_size   ,  shuffle=_SCREAMING_SNAKE_CASE		)
     UpperCAmelCase			:      int			=			len(self.train_dataloader().dataset		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE = False		)						->   Optional[Any]:





    '''simple docstring'''




    raise NotImplementedError("""You must implement this for your task"""		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   str:





    '''simple docstring'''




    return self.train_loader

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   List[Any]:





    '''simple docstring'''




    return self.get_dataloader("""dev"""   ,  self.hparams.eval_batch_size   ,  shuffle=_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Dict:





    '''simple docstring'''




    return self.get_dataloader("""test"""   ,  self.hparams.eval_batch_size   ,  shuffle=_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   Union[str, Any]:





    '''simple docstring'''




    return os.path.join(
        self.hparams.data_dir   ,  """cached_{}_{}_{}""".format(
            _SCREAMING_SNAKE_CASE   ,  list(filter(_SCREAMING_SNAKE_CASE   ,  self.hparams.model_name_or_path.split("""/"""		)		)		).pop()   ,  str(self.hparams.max_seq_length		)   ,  )   ,  )

   @pl.utilities.rank_zero_only
   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   None:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			self.output_dir.joinpath("""best_tfmr"""		)
    UpperCAmelCase			:      int			=			self.step_count
    self.model.save_pretrained(_SCREAMING_SNAKE_CASE		)
    self.tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE		)




   @staticmethod
   def 			SCREAMING_SNAKE_CASE  (     _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   Optional[Any]:





    '''simple docstring'''




    parser.add_argument(
        """--model_name_or_path"""   ,  default=_SCREAMING_SNAKE_CASE   ,  type=_SCREAMING_SNAKE_CASE   ,  required=_SCREAMING_SNAKE_CASE   ,  help="""Path to pretrained model or model identifier from huggingface.co/models"""   ,  )
    parser.add_argument(
        """--config_name"""   ,  default=""""""   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Pretrained config name or path if not the same as model_name"""		)
    parser.add_argument(
        """--tokenizer_name"""   ,  default=_SCREAMING_SNAKE_CASE   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Pretrained tokenizer name or path if not the same as model_name"""   ,  )
    parser.add_argument(
        """--cache_dir"""   ,  default=str(Path(_SCREAMING_SNAKE_CASE		).parent / """test_run""" / """cache"""		)   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Where do you want to store the pre-trained models downloaded from huggingface.co"""   ,  )
    parser.add_argument(
        """--encoder_layerdrop"""   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Encoder layer dropout probability (Optional). Goes into model.config"""   ,  )
    parser.add_argument(
        """--decoder_layerdrop"""   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Decoder layer dropout probability (Optional). Goes into model.config"""   ,  )
    parser.add_argument(
        """--dropout"""   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Dropout probability (Optional). Goes into model.config"""   ,  )
    parser.add_argument(
        """--attention_dropout"""   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Attention dropout probability (Optional). Goes into model.config"""   ,  )
    parser.add_argument("""--learning_rate"""   ,  default=5E-5   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""The initial learning rate for Adam."""		)
    parser.add_argument(
        """--lr_scheduler"""   ,  default="""linear"""   ,  choices=_SCREAMING_SNAKE_CASE   ,  metavar=_SCREAMING_SNAKE_CASE   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Learning rate scheduler"""   ,  )
    parser.add_argument("""--weight_decay"""   ,  default=0.0   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Weight decay if we apply some."""		)
    parser.add_argument("""--adam_epsilon"""   ,  default=1E-8   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Epsilon for Adam optimizer."""		)
    parser.add_argument("""--warmup_steps"""   ,  default=0   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""Linear warmup over warmup_steps."""		)
    parser.add_argument("""--num_workers"""   ,  default=4   ,  type=_SCREAMING_SNAKE_CASE   ,  help="""kwarg passed to DataLoader"""		)
    parser.add_argument("""--num_train_epochs"""   ,  dest="""max_epochs"""   ,  default=3   ,  type=_SCREAMING_SNAKE_CASE		)
    parser.add_argument("""--train_batch_size"""   ,  default=32   ,  type=_SCREAMING_SNAKE_CASE		)
    parser.add_argument("""--eval_batch_size"""   ,  default=32   ,  type=_SCREAMING_SNAKE_CASE		)
    parser.add_argument("""--adafactor"""   ,  action="""store_true"""		)



class    SCREAMING_SNAKE_CASE__						(      pl.Callback				):

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   List[str]:





    '''simple docstring'''




    if (
        trainer.is_global_zero and trainer.global_rank == 0
    ):  # we initialize the retriever only on master worker with RAY. In new pytorch-lightning accelorators are removed.
     pl_module.model.rag.retriever.init_retrieval()  # better to use hook functions.



class    SCREAMING_SNAKE_CASE__						(      pl.Callback				):

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   Optional[int]:





    '''simple docstring'''




    for name, param in pl_module.model.rag.named_parameters():
     if param.grad is None:
      print(_SCREAMING_SNAKE_CASE		)



class    SCREAMING_SNAKE_CASE__						(      pl.Callback				):

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   Union[str, Any]:





    '''simple docstring'''




    UpperCAmelCase			:      Any			=			trainer.lr_schedulers[0]["""scheduler"""]
    UpperCAmelCase			:      int			=			{F"lr_group_{i}": lr for i, lr in enumerate(lr_scheduler.get_lr()		)}
    pl_module.logger.log_metrics(_SCREAMING_SNAKE_CASE		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   Tuple:





    '''simple docstring'''




    rank_zero_info("""***** Validation results *****"""		)
    UpperCAmelCase			:      Any			=			trainer.callback_metrics
    # Log results
    for key in sorted(_SCREAMING_SNAKE_CASE		):
     if key not in ["log", "progress_bar"]:
      rank_zero_info("""{} = {}\n""".format(_SCREAMING_SNAKE_CASE   ,  str(metrics[key]		)		)		)




   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)						->   str:





    '''simple docstring'''




    rank_zero_info("""***** Test results *****"""		)
    UpperCAmelCase			:      Dict			=			trainer.callback_metrics
    # Log and save results to file
    UpperCAmelCase			:      Optional[Any]			=			os.path.join(pl_module.hparams.output_dir   ,  """test_results.txt"""		)
    with open(_SCREAMING_SNAKE_CASE   ,  """w"""		) as writer:
     for key in sorted(_SCREAMING_SNAKE_CASE		):
      if key not in ["log", "progress_bar"]:
       rank_zero_info("""{} = {}\n""".format(_SCREAMING_SNAKE_CASE   ,  str(metrics[key]		)		)		)
       writer.write("""{} = {}\n""".format(_SCREAMING_SNAKE_CASE   ,  str(metrics[key]		)		)		)

def 			_snake_case		(		UpperCamelCase		:	Any    ,	UpperCamelCase		:	List[Any]		):
 #  To allow all pl args uncomment the following line
 #  parser = pl.Trainer.add_argparse_args(parser)
 parser.add_argument(
     """--output_dir"""    ,	default=str(Path(UpperCamelCase		).parent / """test_run""" / """model_checkpoints"""		)    ,	type=UpperCamelCase    ,	help="""The output directory where the model predictions and checkpoints will be written."""    ,	)
 parser.add_argument(
     """--fp16"""    ,	action="""store_true"""    ,	help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""    ,	)

 parser.add_argument(
     """--fp16_opt_level"""    ,	type=UpperCamelCase    ,	default="""O2"""    ,	help=(
         """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
         """See details at https://nvidia.github.io/apex/amp.html"""
     )    ,	)
 parser.add_argument("""--n_tpu_cores"""    ,	dest="""tpu_cores"""    ,	type=UpperCamelCase		)
 parser.add_argument("""--max_grad_norm"""    ,	dest="""gradient_clip_val"""    ,	default=1.0    ,	type=UpperCamelCase    ,	help="""Max gradient norm"""		)
 parser.add_argument("""--do_train"""    ,	action="""store_true"""    ,	help="""Whether to run training."""		)
 parser.add_argument("""--do_predict"""    ,	action="""store_true"""    ,	help="""Whether to run predictions on the test set."""		)
 parser.add_argument(
     """--gradient_accumulation_steps"""    ,	dest="""accumulate_grad_batches"""    ,	type=UpperCamelCase    ,	default=1    ,	help="""Number of updates steps to accumulate before performing a backward/update pass."""    ,	)
 parser.add_argument("""--seed"""    ,	type=UpperCamelCase    ,	default=42    ,	help="""random seed for initialization"""		)
 parser.add_argument(
     """--data_dir"""    ,	default=str(Path(UpperCamelCase		).parent / """test_run""" / """dummy-train-data"""		)    ,	type=UpperCamelCase    ,	help="""The input data dir. Should contain the training files for the CoNLL-2003 NER task."""    ,	)

def 			_snake_case		(		UpperCamelCase		:	BaseTransformer    ,	UpperCamelCase		:	argparse.Namespace    ,	UpperCamelCase		:	Optional[Any]=None    ,	UpperCamelCase		:	Dict=True    ,	UpperCamelCase		:	Dict=[]    ,	UpperCamelCase		:	Union[str, Any]=None    ,	UpperCamelCase		:	Tuple=None    ,	**UpperCamelCase		:	int    ,	):
 pl.seed_everything(args.seed		)

 # init model
 UpperCAmelCase			:      str			=			Path(model.hparams.output_dir		)
 odir.mkdir(exist_ok=UpperCamelCase		)

 # add custom checkpoints
 if checkpoint_callback is None:
  UpperCAmelCase			:      List[str]			=			pl.callbacks.ModelCheckpoint(
      filepath=args.output_dir    ,	prefix="""checkpoint"""    ,	monitor="""val_loss"""    ,	mode="""min"""    ,	save_top_k=1		)
 if early_stopping_callback:
  extra_callbacks.append(UpperCamelCase		)
 if logging_callback is None:
  UpperCAmelCase			:      Tuple			=			LoggingCallback()

 UpperCAmelCase			:      List[Any]			=			{}

 if args.fpaa:
  UpperCAmelCase			:      str			=			16

 if args.gpus > 1:
  UpperCAmelCase			:      Optional[int]			=			"""auto"""
  UpperCAmelCase			:      str			=			"""ddp"""

 UpperCAmelCase			:      Optional[Any]			=			args.accumulate_grad_batches
 UpperCAmelCase			:      int			=			None
 UpperCAmelCase			:      str			=			"""auto"""

 UpperCAmelCase			:      List[Any]			=			pl.Trainer.from_argparse_args(
     UpperCamelCase    ,	weights_summary=UpperCamelCase    ,	callbacks=[logging_callback] + extra_callbacks + [InitCallback()] + [checkpoint_callback]    ,	logger=UpperCamelCase    ,	val_check_interval=1    ,	num_sanity_val_steps=2    ,	**UpperCamelCase    ,	)

 if args.do_train:
  trainer.fit(UpperCamelCase		)

 else:
  print("""RAG modeling tests with new set functions successfuly executed!"""		)
 return trainer





"""simple docstring"""
import json
import os
import unittest

from transformers.models.xlm.tokenization_xlm import VOCAB_FILES_NAMES, XLMTokenizer
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class    SCREAMING_SNAKE_CASE__						(      UpperCAmelCase__   ,    unittest.TestCase				):
   __lowerCAmelCase						:     Any		=		XLMTokenizer
   __lowerCAmelCase						:     List[Any]		=		False

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    super().setUp()

    # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
    UpperCAmelCase			:      Optional[int]			=			[
        """l""",
        """o""",
        """w""",
        """e""",
        """r""",
        """s""",
        """t""",
        """i""",
        """d""",
        """n""",
        """w</w>""",
        """r</w>""",
        """t</w>""",
        """lo""",
        """low""",
        """er</w>""",
        """low</w>""",
        """lowest</w>""",
        """newer</w>""",
        """wider</w>""",
        """<unk>""",
    ]
    UpperCAmelCase			:      str			=			dict(zip(_SCREAMING_SNAKE_CASE   ,  range(len(_SCREAMING_SNAKE_CASE		)		)		)		)
    UpperCAmelCase			:      List[Any]			=			["""l o 123""", """lo w 1456""", """e r</w> 1789""", """"""]

    UpperCAmelCase			:      Optional[Any]			=			os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES["""vocab_file"""]		)
    UpperCAmelCase			:      Optional[Any]			=			os.path.join(self.tmpdirname   ,  VOCAB_FILES_NAMES["""merges_file"""]		)
    with open(self.vocab_file   ,  """w"""		) as fp:
     fp.write(json.dumps(_SCREAMING_SNAKE_CASE		)		)
    with open(self.merges_file   ,  """w"""		) as fp:
     fp.write("""\n""".join(_SCREAMING_SNAKE_CASE		)		)

   def 			SCREAMING_SNAKE_CASE  (     self   ,  _SCREAMING_SNAKE_CASE		)						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      Optional[int]			=			"""lower newer"""
    UpperCAmelCase			:      Dict			=			"""lower newer"""
    return input_text, output_text

   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Optional[int]:





    '''simple docstring'''




    UpperCAmelCase			:      Union[str, Any]			=			XLMTokenizer(self.vocab_file   ,  self.merges_file		)

    UpperCAmelCase			:      Optional[int]			=			"""lower"""
    UpperCAmelCase			:      Optional[int]			=			["""low""", """er</w>"""]
    UpperCAmelCase			:      List[Any]			=			tokenizer.tokenize(_SCREAMING_SNAKE_CASE		)
    self.assertListEqual(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Optional[int]			=			tokens + ["""<unk>"""]
    UpperCAmelCase			:      List[Any]			=			[14, 15, 20]
    self.assertListEqual(tokenizer.convert_tokens_to_ids(_SCREAMING_SNAKE_CASE		)   ,  _SCREAMING_SNAKE_CASE		)




   @slow
   def 			SCREAMING_SNAKE_CASE  (     self		)						->   Any:





    '''simple docstring'''




    UpperCAmelCase			:      List[Any]			=			XLMTokenizer.from_pretrained("""xlm-mlm-en-2048"""		)

    UpperCAmelCase			:      int			=			tokenizer.encode("""sequence builders"""   ,  add_special_tokens=_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      List[Any]			=			tokenizer.encode("""multi-sequence build"""   ,  add_special_tokens=_SCREAMING_SNAKE_CASE		)

    UpperCAmelCase			:      Tuple			=			tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE		)
    UpperCAmelCase			:      str			=			tokenizer.build_inputs_with_special_tokens(_SCREAMING_SNAKE_CASE   ,  _SCREAMING_SNAKE_CASE		)

    assert encoded_sentence == [0] + text + [1]
    assert encoded_pair == [0] + text + [1] + text_a + [1]
