






'''simple docstring'''



import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
      from transformers.trainer import DEFAULT_CALLBACKS

      from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel
class UpperCAmelCase_       (     __A					):




      """simple docstring"""




      def __init__(			self      :		Optional[Any]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=[]






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		List[Any]  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		Any  ,   **UpperCAmelCase      :		Tuple		)	->	Optional[Any]:


       '''simple docstring'''

       self.events.append('''on_init_end'''		)






      def 					A__   (			self      :		Union[str, Any]  ,   UpperCAmelCase      :		List[Any]  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		Any  ,   **UpperCAmelCase      :		Dict		)	->	int:


       '''simple docstring'''

       self.events.append('''on_train_begin'''		)






      def 					A__   (			self      :		Dict  ,   UpperCAmelCase      :		Optional[Any]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		Union[str, Any]  ,   **UpperCAmelCase      :		Tuple		)	->	Optional[Any]:


       '''simple docstring'''

       self.events.append('''on_train_end'''		)






      def 					A__   (			self      :		Any  ,   UpperCAmelCase      :		str  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		Dict  ,   **UpperCAmelCase      :		Any		)	->	Dict:


       '''simple docstring'''

       self.events.append('''on_epoch_begin'''		)






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		List[Any]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		List[str]  ,   **UpperCAmelCase      :		List[Any]		)	->	List[str]:


       '''simple docstring'''

       self.events.append('''on_epoch_end'''		)






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		Optional[Any]  ,   **UpperCAmelCase      :		Optional[int]		)	->	int:


       '''simple docstring'''

       self.events.append('''on_step_begin'''		)






      def 					A__   (			self      :		List[Any]  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		List[Any]  ,   UpperCAmelCase      :		Tuple  ,   **UpperCAmelCase      :		List[Any]		)	->	Any:


       '''simple docstring'''

       self.events.append('''on_step_end'''		)






      def 					A__   (			self      :		Optional[int]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		Optional[Any]  ,   UpperCAmelCase      :		Union[str, Any]  ,   **UpperCAmelCase      :		Tuple		)	->	Dict:


       '''simple docstring'''

       self.events.append('''on_evaluate'''		)






      def 					A__   (			self      :		int  ,   UpperCAmelCase      :		Any  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		List[str]  ,   **UpperCAmelCase      :		str		)	->	List[str]:


       '''simple docstring'''

       self.events.append('''on_predict'''		)






      def 					A__   (			self      :		Optional[Any]  ,   UpperCAmelCase      :		Tuple  ,   UpperCAmelCase      :		Optional[int]  ,   UpperCAmelCase      :		int  ,   **UpperCAmelCase      :		Dict		)	->	Union[str, Any]:


       '''simple docstring'''

       self.events.append('''on_save'''		)






      def 					A__   (			self      :		int  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		int  ,   UpperCAmelCase      :		Any  ,   **UpperCAmelCase      :		int		)	->	str:


       '''simple docstring'''

       self.events.append('''on_log'''		)






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		List[str]  ,   UpperCAmelCase      :		str  ,   **UpperCAmelCase      :		Any		)	->	Tuple:


       '''simple docstring'''

       self.events.append('''on_prediction_step'''		)





@require_torch
class UpperCAmelCase_       (     unittest.TestCase					):




      """simple docstring"""




      def 					A__   (			self      :		Optional[Any]		)	->	int:


       '''simple docstring'''

       lowercase       : Optional[int]						=tempfile.mkdtemp()






      def 					A__   (			self      :		Optional[int]		)	->	Union[str, Any]:


       '''simple docstring'''

       shutil.rmtree(self.output_dir		)






      def 					A__   (			self      :		str  ,   UpperCAmelCase      :		List[Any]=0  ,   UpperCAmelCase      :		Tuple=0  ,   UpperCAmelCase      :		Dict=64  ,   UpperCAmelCase      :		List[Any]=64  ,   UpperCAmelCase      :		Optional[int]=None  ,   UpperCAmelCase      :		Optional[Any]=False  ,   **UpperCAmelCase      :		Optional[int]		)	->	Union[str, Any]:


       '''simple docstring'''

       lowercase       : Union[str, Any]						=RegressionDataset(length=UpperCAmelCase		)
       lowercase       : Any						=RegressionDataset(length=UpperCAmelCase		)
       lowercase       : Optional[int]						=RegressionModelConfig(a=UpperCAmelCase  ,   b=UpperCAmelCase		)
       lowercase       : Any						=RegressionPreTrainedModel(UpperCAmelCase		)

       lowercase       : Dict						=TrainingArguments(self.output_dir  ,   disable_tqdm=UpperCAmelCase  ,   report_to=[]  ,   **UpperCAmelCase		)
       return Trainer(
           UpperCAmelCase  ,   UpperCAmelCase  ,   train_dataset=UpperCAmelCase  ,   eval_dataset=UpperCAmelCase  ,   callbacks=UpperCAmelCase  ,   )






      def 					A__   (			self      :		Tuple  ,   UpperCAmelCase      :		Union[str, Any]  ,   UpperCAmelCase      :		List[str]		)	->	Tuple:


       '''simple docstring'''

       self.assertEqual(len(UpperCAmelCase		)  ,   len(UpperCAmelCase		)		)

       # Order doesn't matter
       lowercase       : Tuple						=sorted(UpperCAmelCase  ,   key=lambda UpperCAmelCase		: cb.__name__ if isinstance(UpperCAmelCase  ,   UpperCAmelCase		) else cb.__class__.__name__		)
       lowercase       : Any						=sorted(UpperCAmelCase  ,   key=lambda UpperCAmelCase		: cb.__name__ if isinstance(UpperCAmelCase  ,   UpperCAmelCase		) else cb.__class__.__name__		)

       for cba, cba in zip(UpperCAmelCase  ,   UpperCAmelCase		):
        if isinstance(UpperCAmelCase  ,   UpperCAmelCase		) and isinstance(UpperCAmelCase  ,   UpperCAmelCase		):
         self.assertEqual(UpperCAmelCase  ,   UpperCAmelCase		)
        elif isinstance(UpperCAmelCase  ,   UpperCAmelCase		) and not isinstance(UpperCAmelCase  ,   UpperCAmelCase		):
         self.assertEqual(UpperCAmelCase  ,   cba.__class__		)
        elif not isinstance(UpperCAmelCase  ,   UpperCAmelCase		) and isinstance(UpperCAmelCase  ,   UpperCAmelCase		):
         self.assertEqual(cba.__class__  ,   UpperCAmelCase		)
        else:
         self.assertEqual(UpperCAmelCase  ,   UpperCAmelCase		)






      def 					A__   (			self      :		Dict  ,   UpperCAmelCase      :		Optional[Any]		)	->	Any:


       '''simple docstring'''

       lowercase       : Tuple						=['''on_init_end''', '''on_train_begin''']
       lowercase       : List[str]						=0
       lowercase       : Union[str, Any]						=len(trainer.get_eval_dataloader()		)
       lowercase       : Optional[Any]						=['''on_prediction_step'''] * len(trainer.get_eval_dataloader()		) + ['''on_log''', '''on_evaluate''']
       for _ in range(trainer.state.num_train_epochs		):
        expected_events.append('''on_epoch_begin'''		)
        for _ in range(UpperCAmelCase		):
         step += 1
         expected_events += ["on_step_begin", "on_step_end"]
         if step % trainer.args.logging_steps == 0:
          expected_events.append('''on_log'''		)
         if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
          expected_events += evaluation_events.copy()
         if step % trainer.args.save_steps == 0:
          expected_events.append('''on_save'''		)
        expected_events.append('''on_epoch_end'''		)
        if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
         expected_events += evaluation_events.copy()
       expected_events += ["on_log", "on_train_end"]
       return expected_events






      def 					A__   (			self      :		Any		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : List[str]						=self.get_trainer()
       lowercase       : Dict						=DEFAULT_CALLBACKS.copy() + [ProgressCallback]
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)

       # Callbacks passed at init are added to the default callbacks
       lowercase       : List[Any]						=self.get_trainer(callbacks=[MyTestTrainerCallback]		)
       expected_callbacks.append(UpperCAmelCase		)
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)

       # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
       lowercase       : Tuple						=self.get_trainer(disable_tqdm=UpperCAmelCase		)
       lowercase       : str						=DEFAULT_CALLBACKS.copy() + [PrinterCallback]
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)






      def 					A__   (			self      :		int		)	->	Optional[Any]:


       '''simple docstring'''

       lowercase       : Any						=DEFAULT_CALLBACKS.copy() + [ProgressCallback]
       lowercase       : Optional[int]						=self.get_trainer()

       # We can add, pop, or remove by class name
       trainer.remove_callback(UpperCAmelCase		)
       expected_callbacks.remove(UpperCAmelCase		)
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)

       lowercase       : Union[str, Any]						=self.get_trainer()
       lowercase       : Union[str, Any]						=trainer.pop_callback(UpperCAmelCase		)
       self.assertEqual(cb.__class__  ,   UpperCAmelCase		)
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)

       trainer.add_callback(UpperCAmelCase		)
       expected_callbacks.insert(0  ,   UpperCAmelCase		)
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)

       # We can also add, pop, or remove by instance
       lowercase       : Optional[int]						=self.get_trainer()
       lowercase       : str						=trainer.callback_handler.callbacks[0]
       trainer.remove_callback(UpperCAmelCase		)
       expected_callbacks.remove(UpperCAmelCase		)
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)

       lowercase       : Union[str, Any]						=self.get_trainer()
       lowercase       : List[Any]						=trainer.callback_handler.callbacks[0]
       lowercase       : Optional[int]						=trainer.pop_callback(UpperCAmelCase		)
       self.assertEqual(UpperCAmelCase  ,   UpperCAmelCase		)
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)

       trainer.add_callback(UpperCAmelCase		)
       expected_callbacks.insert(0  ,   UpperCAmelCase		)
       self.check_callbacks_equality(trainer.callback_handler.callbacks  ,   UpperCAmelCase		)






      def 					A__   (			self      :		str		)	->	Optional[int]:


       '''simple docstring'''

       import warnings

       # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
       warnings.simplefilter(action='''ignore'''  ,   category=UpperCAmelCase		)

       lowercase       : List[Any]						=self.get_trainer(callbacks=[MyTestTrainerCallback]		)
       trainer.train()
       lowercase       : List[Any]						=trainer.callback_handler.callbacks[-2].events
       self.assertEqual(UpperCAmelCase  ,   self.get_expected_events(UpperCAmelCase		)		)

       # Independent log/save/eval
       lowercase       : int						=self.get_trainer(callbacks=[MyTestTrainerCallback]  ,   logging_steps=5		)
       trainer.train()
       lowercase       : List[str]						=trainer.callback_handler.callbacks[-2].events
       self.assertEqual(UpperCAmelCase  ,   self.get_expected_events(UpperCAmelCase		)		)

       lowercase       : Dict						=self.get_trainer(callbacks=[MyTestTrainerCallback]  ,   save_steps=5		)
       trainer.train()
       lowercase       : Union[str, Any]						=trainer.callback_handler.callbacks[-2].events
       self.assertEqual(UpperCAmelCase  ,   self.get_expected_events(UpperCAmelCase		)		)

       lowercase       : int						=self.get_trainer(callbacks=[MyTestTrainerCallback]  ,   eval_steps=5  ,   evaluation_strategy='''steps'''		)
       trainer.train()
       lowercase       : List[str]						=trainer.callback_handler.callbacks[-2].events
       self.assertEqual(UpperCAmelCase  ,   self.get_expected_events(UpperCAmelCase		)		)

       lowercase       : Optional[Any]						=self.get_trainer(callbacks=[MyTestTrainerCallback]  ,   evaluation_strategy='''epoch'''		)
       trainer.train()
       lowercase       : Optional[int]						=trainer.callback_handler.callbacks[-2].events
       self.assertEqual(UpperCAmelCase  ,   self.get_expected_events(UpperCAmelCase		)		)

       # A bit of everything
       lowercase       : Dict						=self.get_trainer(
           callbacks=[MyTestTrainerCallback]  ,   logging_steps=3  ,   save_steps=10  ,   eval_steps=5  ,   evaluation_strategy='''steps'''  ,   )
       trainer.train()
       lowercase       : Optional[Any]						=trainer.callback_handler.callbacks[-2].events
       self.assertEqual(UpperCAmelCase  ,   self.get_expected_events(UpperCAmelCase		)		)

       # warning should be emitted for duplicated callbacks
       with patch('''transformers.trainer_callback.logger.warning'''		) as warn_mock:
        lowercase       : Optional[Any]						=self.get_trainer(
            callbacks=[MyTestTrainerCallback, MyTestTrainerCallback]  ,   )
        assert str(UpperCAmelCase		) in warn_mock.call_args[0][0]











'''simple docstring'''



import json
import os
import unittest

from transformers import BatchEncoding, MvpTokenizer, MvpTokenizerFast
from transformers.models.roberta.tokenization_roberta import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers, require_torch
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin, filter_roberta_detectors
@require_tokenizers
class UpperCAmelCase_       (     __A					,							unittest.TestCase					):




      """simple docstring"""




      UpperCamelCase_             =      MvpTokenizer
      UpperCamelCase_             =      MvpTokenizerFast
      UpperCamelCase_             =      True
      UpperCamelCase_             =      filter_roberta_detectors






      def 					A__   (			self      :		Union[str, Any]		)	->	List[str]:


       '''simple docstring'''

       super().setUp()
       lowercase       : Dict						=[
           '''l''',
           '''o''',
           '''w''',
           '''e''',
           '''r''',
           '''s''',
           '''t''',
           '''i''',
           '''d''',
           '''n''',
           '''\u0120''',
           '''\u0120l''',
           '''\u0120n''',
           '''\u0120lo''',
           '''\u0120low''',
           '''er''',
           '''\u0120lowest''',
           '''\u0120newer''',
           '''\u0120wider''',
           '''<unk>''',
       ]
       lowercase       : Tuple						=dict(zip(UpperCAmelCase  ,   range(len(UpperCAmelCase		)		)		)		)
       lowercase       : Optional[Any]						=['''#version: 0.2''', '''\u0120 l''', '''\u0120l o''', '''\u0120lo w''', '''e r''', '''''']
       lowercase       : List[Any]						={'''unk_token''': '''<unk>'''}

       lowercase       : Dict						=os.path.join(self.tmpdirname  ,   VOCAB_FILES_NAMES['''vocab_file''']		)
       lowercase       : Dict						=os.path.join(self.tmpdirname  ,   VOCAB_FILES_NAMES['''merges_file''']		)
       with open(self.vocab_file  ,   '''w'''  ,   encoding='''utf-8'''		) as fp:
        fp.write(json.dumps(UpperCAmelCase		) + '''\n'''		)
       with open(self.merges_file  ,   '''w'''  ,   encoding='''utf-8'''		) as fp:
        fp.write('''\n'''.join(UpperCAmelCase		)		)






      def 					A__   (			self      :		Union[str, Any]  ,   **UpperCAmelCase      :		Dict		)	->	Tuple:


       '''simple docstring'''

       kwargs.update(self.special_tokens_map		)
       return self.tokenizer_class.from_pretrained(self.tmpdirname  ,   **UpperCAmelCase		)






      def 					A__   (			self      :		List[str]  ,   **UpperCAmelCase      :		Any		)	->	Tuple:


       '''simple docstring'''

       kwargs.update(self.special_tokens_map		)
       return self.rust_tokenizer_class.from_pretrained(self.tmpdirname  ,   **UpperCAmelCase		)






      def 					A__   (			self      :		Tuple  ,   UpperCAmelCase      :		Any		)	->	Union[str, Any]:


       '''simple docstring'''

       return "lower newer", "lower newer"






      @cached_property
      def 					A__   (			self      :		Tuple		)	->	List[Any]:


       '''simple docstring'''

       return MvpTokenizer.from_pretrained('''RUCAIBox/mvp'''		)






      @cached_property
      def 					A__   (			self      :		Any		)	->	int:


       '''simple docstring'''

       return MvpTokenizerFast.from_pretrained('''RUCAIBox/mvp'''		)






      @require_torch
      def 					A__   (			self      :		Union[str, Any]		)	->	Optional[int]:


       '''simple docstring'''

       lowercase       : Optional[int]						=['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
       lowercase       : List[str]						=[0, 250, 251, 1_7818, 13, 3_9186, 1938, 4, 2]

       for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
        lowercase       : Union[str, Any]						=tokenizer(UpperCAmelCase  ,   max_length=len(UpperCAmelCase		)  ,   padding=UpperCAmelCase  ,   return_tensors='''pt'''		)
        self.assertIsInstance(UpperCAmelCase  ,   UpperCAmelCase		)

        self.assertEqual((2, 9)  ,   batch.input_ids.shape		)
        self.assertEqual((2, 9)  ,   batch.attention_mask.shape		)
        lowercase       : Union[str, Any]						=batch.input_ids.tolist()[0]
        self.assertListEqual(UpperCAmelCase  ,   UpperCAmelCase		)
        # Test that special tokens are reset






      @require_torch
      def 					A__   (			self      :		Tuple		)	->	Any:


       '''simple docstring'''

       lowercase       : Any						=['''A long paragraph for summarization.''', '''Another paragraph for summarization.''']
       for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
        lowercase       : Dict						=tokenizer(UpperCAmelCase  ,   padding=UpperCAmelCase  ,   return_tensors='''pt'''		)
        # check if input_ids are returned and no labels
        self.assertIn('''input_ids'''  ,   UpperCAmelCase		)
        self.assertIn('''attention_mask'''  ,   UpperCAmelCase		)
        self.assertNotIn('''labels'''  ,   UpperCAmelCase		)
        self.assertNotIn('''decoder_attention_mask'''  ,   UpperCAmelCase		)






      @require_torch
      def 					A__   (			self      :		Dict		)	->	int:


       '''simple docstring'''

       lowercase       : int						=[
           '''Summary of the text.''',
           '''Another summary.''',
       ]
       for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
        lowercase       : Optional[Any]						=tokenizer(text_target=UpperCAmelCase  ,   max_length=32  ,   padding='''max_length'''  ,   return_tensors='''pt'''		)
        self.assertEqual(32  ,   targets['''input_ids'''].shape[1]		)






      @require_torch
      def 					A__   (			self      :		Union[str, Any]		)	->	Any:


       '''simple docstring'''

       for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
        lowercase       : Union[str, Any]						=tokenizer(
            ['''I am a small frog''' * 1024, '''I am a small frog''']  ,   padding=UpperCAmelCase  ,   truncation=UpperCAmelCase  ,   return_tensors='''pt'''		)
        self.assertIsInstance(UpperCAmelCase  ,   UpperCAmelCase		)
        self.assertEqual(batch.input_ids.shape  ,   (2, 1024)		)






      @require_torch
      def 					A__   (			self      :		str		)	->	List[Any]:


       '''simple docstring'''

       lowercase       : Optional[Any]						=['''A long paragraph for summarization.''']
       lowercase       : List[Any]						=[
           '''Summary of the text.''',
       ]
       for tokenizer in [self.default_tokenizer, self.default_tokenizer_fast]:
        lowercase       : List[str]						=tokenizer(UpperCAmelCase  ,   text_target=UpperCAmelCase  ,   return_tensors='''pt'''		)
        lowercase       : Optional[int]						=inputs['''input_ids''']
        lowercase       : Optional[Any]						=inputs['''labels''']
        self.assertTrue((input_ids[:, 0] == tokenizer.bos_token_id).all().item()		)
        self.assertTrue((labels[:, 0] == tokenizer.bos_token_id).all().item()		)
        self.assertTrue((input_ids[:, -1] == tokenizer.eos_token_id).all().item()		)
        self.assertTrue((labels[:, -1] == tokenizer.eos_token_id).all().item()		)






      def 					A__   (			self      :		List[Any]		)	->	List[Any]:


       '''simple docstring'''

       pass






      def 					A__   (			self      :		List[str]		)	->	Tuple:


       '''simple docstring'''

       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
        with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'		):
         lowercase       : Union[str, Any]						=self.rust_tokenizer_class.from_pretrained(UpperCAmelCase  ,   **UpperCAmelCase		)
         lowercase       : Tuple						=self.tokenizer_class.from_pretrained(UpperCAmelCase  ,   **UpperCAmelCase		)
         lowercase       : Optional[Any]						='''A, <mask> AllenNLP sentence.'''
         lowercase       : int						=tokenizer_r.encode_plus(UpperCAmelCase  ,   add_special_tokens=UpperCAmelCase  ,   return_token_type_ids=UpperCAmelCase		)
         lowercase       : List[Any]						=tokenizer_p.encode_plus(UpperCAmelCase  ,   add_special_tokens=UpperCAmelCase  ,   return_token_type_ids=UpperCAmelCase		)

         # token_type_ids should put 0 everywhere
         self.assertEqual(sum(tokens_r['''token_type_ids''']		)  ,   sum(tokens_p['''token_type_ids''']		)		)

         # attention_mask should put 1 everywhere, so sum over length should be 1
         self.assertEqual(
             sum(tokens_r['''attention_mask''']		) / len(tokens_r['''attention_mask''']		)  ,   sum(tokens_p['''attention_mask''']		) / len(tokens_p['''attention_mask''']		)  ,   )

         lowercase       : Any						=tokenizer_r.convert_ids_to_tokens(tokens_r['''input_ids''']		)
         lowercase       : str						=tokenizer_p.convert_ids_to_tokens(tokens_p['''input_ids''']		)

         # Rust correctly handles the space before the mask while python doesnt
         self.assertSequenceEqual(tokens_p['''input_ids''']  ,   [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]		)
         self.assertSequenceEqual(tokens_r['''input_ids''']  ,   [0, 250, 6, 5_0264, 3823, 487, 2_1992, 3645, 4, 2]		)

         self.assertSequenceEqual(
             UpperCAmelCase  ,   ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']		)
         self.assertSequenceEqual(
             UpperCAmelCase  ,   ['''<s>''', '''A''', ''',''', '''<mask>''', '''ĠAllen''', '''N''', '''LP''', '''Ġsentence''', '''.''', '''</s>''']		)



