


import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__					:     Any       =     get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
@require_tokenizers
class 		_SCREAMING_SNAKE_CASE     (						A							,	unittest.TestCase     ):
     __SCREAMING_SNAKE_CASE     					=    ReformerTokenizer
     __SCREAMING_SNAKE_CASE     					=    ReformerTokenizerFast
     __SCREAMING_SNAKE_CASE     					=    True
     __SCREAMING_SNAKE_CASE     					=    False
     __SCREAMING_SNAKE_CASE     					=    True


     def        __snake_case(  self    ):
       super().setUp()

       _UpperCAmelCase		:							Optional[Any]							=     ReformerTokenizer(lowercase__							,					keep_accents=lowercase__    )
       tokenizer.save_pretrained(self.tmpdirname    )


     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Dict							=     """<s>"""
       _UpperCAmelCase		:							Dict							=     1

       self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase__    )							,					lowercase__    )
       self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase__    )							,					lowercase__    )


     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Any							=     list(self.get_tokenizer().get_vocab().keys()    )

       self.assertEqual(vocab_keys[0]							,					"""<unk>"""    )
       self.assertEqual(vocab_keys[1]							,					"""<s>"""    )
       self.assertEqual(vocab_keys[-1]							,					"""j"""    )
       self.assertEqual(len(lowercase__    )							,					10_00    )


     def        __snake_case(  self    ):
       self.assertEqual(self.get_tokenizer().vocab_size							,					10_00    )


     def        __snake_case(  self    ):
       if not self.test_rust_tokenizer:
         return

       _UpperCAmelCase		:							Dict							=     self.get_tokenizer()
       _UpperCAmelCase		:							Union[str, Any]							=     self.get_rust_tokenizer()

       _UpperCAmelCase		:							List[str]							=     """I was born in 92000, and this is falsé."""

       _UpperCAmelCase		:							Optional[Any]							=     tokenizer.tokenize(lowercase__    )
       _UpperCAmelCase		:							Dict							=     rust_tokenizer.tokenize(lowercase__    )
       self.assertListEqual(lowercase__							,					lowercase__    )

       _UpperCAmelCase		:							int							=     tokenizer.encode(lowercase__							,					add_special_tokens=lowercase__    )
       _UpperCAmelCase		:							List[Any]							=     rust_tokenizer.encode(lowercase__							,					add_special_tokens=lowercase__    )
       self.assertListEqual(lowercase__							,					lowercase__    )

       _UpperCAmelCase		:							List[str]							=     self.get_rust_tokenizer()
       _UpperCAmelCase		:							Tuple							=     tokenizer.encode(lowercase__    )
       _UpperCAmelCase		:							Tuple							=     rust_tokenizer.encode(lowercase__    )
       self.assertListEqual(lowercase__							,					lowercase__    )


     def        __snake_case(  self							,					A_=15    ):
       for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
         with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''    ):
           _UpperCAmelCase		:							List[str]							=     self.rust_tokenizer_class.from_pretrained(lowercase__							,					**lowercase__    )

           # Simple input
           _UpperCAmelCase		:							List[str]							=     """This is a simple input"""
           _UpperCAmelCase		:							int							=     ["""This is a simple input 1""", """This is a simple input 2"""]
           _UpperCAmelCase		:							int							=     ("""This is a simple input""", """This is a pair""")
           _UpperCAmelCase		:							Dict							=     [
               ("""This is a simple input 1""", """This is a simple input 2"""),
               ("""This is a simple pair 1""", """This is a simple pair 2"""),
           ]

           # Simple input tests
           self.assertRaises(lowercase__							,					tokenizer_r.encode							,					lowercase__							,					max_length=lowercase__							,					padding="""max_length"""    )

           # Simple input
           self.assertRaises(lowercase__							,					tokenizer_r.encode_plus							,					lowercase__							,					max_length=lowercase__							,					padding="""max_length"""    )

           # Simple input
           self.assertRaises(
               lowercase__							,					tokenizer_r.batch_encode_plus							,					lowercase__							,					max_length=lowercase__							,					padding="""max_length"""							,					)

           # Pair input
           self.assertRaises(lowercase__							,					tokenizer_r.encode							,					lowercase__							,					max_length=lowercase__							,					padding="""max_length"""    )

           # Pair input
           self.assertRaises(lowercase__							,					tokenizer_r.encode_plus							,					lowercase__							,					max_length=lowercase__							,					padding="""max_length"""    )

           # Pair input
           self.assertRaises(
               lowercase__							,					tokenizer_r.batch_encode_plus							,					lowercase__							,					max_length=lowercase__							,					padding="""max_length"""							,					)


     def        __snake_case(  self    ):
       pass


     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Union[str, Any]							=     ReformerTokenizer(lowercase__							,					keep_accents=lowercase__    )

       _UpperCAmelCase		:							str							=     tokenizer.tokenize("""This is a test"""    )
       self.assertListEqual(lowercase__							,					["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]    )

       self.assertListEqual(
           tokenizer.convert_tokens_to_ids(lowercase__    )							,					[2_85, 46, 10, 1_70, 3_82]							,					)

       _UpperCAmelCase		:							Dict							=     tokenizer.tokenize("""I was born in 92000, and this is falsé."""    )
       self.assertListEqual(
           lowercase__							,					[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """9""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """é""",
               """.""",
           ]							,					)
       _UpperCAmelCase		:							List[str]							=     tokenizer.convert_tokens_to_ids(lowercase__    )
       self.assertListEqual(
           lowercase__							,					[8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]							,					)

       _UpperCAmelCase		:							List[str]							=     tokenizer.convert_ids_to_tokens(lowercase__    )
       self.assertListEqual(
           lowercase__							,					[
               SPIECE_UNDERLINE + """I""",
               SPIECE_UNDERLINE + """was""",
               SPIECE_UNDERLINE + """b""",
               """or""",
               """n""",
               SPIECE_UNDERLINE + """in""",
               SPIECE_UNDERLINE + """""",
               """<unk>""",
               """2""",
               """0""",
               """0""",
               """0""",
               """,""",
               SPIECE_UNDERLINE + """and""",
               SPIECE_UNDERLINE + """this""",
               SPIECE_UNDERLINE + """is""",
               SPIECE_UNDERLINE + """f""",
               """al""",
               """s""",
               """<unk>""",
               """.""",
           ]							,					)


     @cached_property
     def        __snake_case(  self    ):
       return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""    )


     @slow
     def        __snake_case(  self    ):
       _UpperCAmelCase		:							Optional[Any]							=     """Hello World!"""
       _UpperCAmelCase		:							Tuple							=     [1_26, 32, 2_62, 1_52, 38, 72, 2_87]

       self.assertListEqual(lowercase__							,					self.big_tokenizer.encode(lowercase__    )    )


     @slow
     def        __snake_case(  self    ):
       _UpperCAmelCase		:							List[str]							=     (
           """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
           """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
       )
       _UpperCAmelCase		:							Any							=     [
           1_08,
           2_65,
           24,
           1_11,
           4,
           2_58,
           1_56,
           35,
           28,
           2_75,
           3,
           2_59,
           2_97,
           2_60,
           84,
           4,
           35,
           1_10,
           44,
           8,
           2_59,
           91,
           2_68,
           21,
           11,
           2_09,
           2_74,
           1_09,
           2_66,
           2_77,
           1_17,
           86,
           93,
           3_15,
           2_58,
           2_78,
           2_58,
           2_77,
           2_58,
           0,
           2_58,
           2_88,
           2_58,
           3_19,
           2_58,
           0,
           2_58,
           0,
           2_58,
           0,
           2_58,
           0,
           2_58,
           2_87,
           2_58,
           3_15,
           2_58,
           2_89,
           2_58,
           2_78,
           99,
           2_69,
           2_66,
           2_62,
           8,
           2_59,
           2_41,
           4,
           2_17,
           2_30,
           2_68,
           2_66,
           55,
           1_68,
           1_06,
           75,
           1_93,
           2_66,
           2_23,
           27,
           49,
           26,
           2_82,
           25,
           2_64,
           2_99,
           19,
           26,
           0,
           2_58,
           2_77,
           1_17,
           86,
           93,
           1_76,
           1_83,
           2_70,
           11,
           2_62,
           42,
           61,
           2_65,
       ]

       self.assertListEqual(lowercase__							,					self.big_tokenizer.encode(lowercase__    )    )


     @require_torch
     @slow
     def        __snake_case(  self    ):
       import torch

       from transformers import ReformerConfig, ReformerModel

       # Build sequence
       _UpperCAmelCase		:							Dict							=     list(self.big_tokenizer.get_vocab().keys()    )[:10]
       _UpperCAmelCase		:							str							=     """ """.join(lowercase__    )
       _UpperCAmelCase		:							Tuple							=     self.big_tokenizer.encode_plus(lowercase__							,					return_tensors="""pt"""    )
       _UpperCAmelCase		:							Optional[Any]							=     self.big_tokenizer.batch_encode_plus([sequence, sequence]							,					return_tensors="""pt"""    )

       _UpperCAmelCase		:							Optional[int]							=     ReformerConfig()
       # The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
       _UpperCAmelCase		:							Union[str, Any]							=     encoded_sequence["""input_ids"""].shape
       _UpperCAmelCase		:							Union[str, Any]							=     ReformerModel(lowercase__    )

       # Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
       assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

       with torch.no_grad():
         model(**lowercase__    )
         model(**lowercase__    )




     @slow
     def        __snake_case(  self    ):
       # fmt: off
       _UpperCAmelCase		:							Union[str, Any]							=     {"""input_ids""": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
       # fmt: on

       # This tokenizer does not know some characters like ")".
       # That is the reason why we use very simple texts here.
       # Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
       _UpperCAmelCase		:							Dict							=     [
           """This is a very simple sentence.""",
           """The quick brown fox jumps over the lazy dog.""",
       ]

       self.tokenizer_integration_test_util(
           expected_encoding=lowercase__							,					model_name="""google/reformer-crime-and-punishment"""							,					revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a"""							,					padding=lowercase__							,					sequences=lowercase__							,					)






def _lowerCamelCase  (		snake_case = 10  ):
							if not isinstance(snake_case     , snake_case  ) or n < 0:
														raise ValueError('Invalid input'  )
							_lowerCAmelCase							=							10**n
							_lowerCAmelCase							=							28_433 * (pow(2     , 7_830_457     , snake_case  )) + 1
							return str(number % modulus  )


if __name__ == "__main__":
				from doctest import testmod

				testmod()
				print(f"""{solution(1_0) = }""")






