





import unittest

from transformers import SPIECE_UNDERLINE, ReformerTokenizer, ReformerTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCAmelCase_							=				get_tests_dir("""fixtures/test_sentencepiece.model""")
@require_sentencepiece
@require_tokenizers
class     UpperCamelCase__					(     lowerCamelCase__				, unittest.TestCase				):






		'''simple docstring'''






		__a       :       Tuple       =	ReformerTokenizer
		__a       :       str       =	ReformerTokenizerFast
		__a       :       Optional[Any]       =	True
		__a       :       str       =	False
		__a       :       Dict       =	True






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									super().setUp()

									lowercase_	:   Union[str, Any]  				=     ReformerTokenizer(snake_case__,   keep_accents=snake_case__  )
									tokenizer.save_pretrained(self.tmpdirname  )






		def 						snake_case__				(							self  )       -> Any:






									"""simple docstring"""
									lowercase_	:   str  				=     """<s>"""
									lowercase_	:   List[str]  				=     1

									self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__  ),   snake_case__  )
									self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__  ),   snake_case__  )






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									lowercase_	:   Any  				=     list(self.get_tokenizer().get_vocab().keys()  )

									self.assertEqual(vocab_keys[0],   """<unk>"""  )
									self.assertEqual(vocab_keys[1],   """<s>"""  )
									self.assertEqual(vocab_keys[-1],   """j"""  )
									self.assertEqual(len(snake_case__  ),   10_00  )






		def 						snake_case__				(							self  )       -> Optional[Any]:






									"""simple docstring"""
									self.assertEqual(self.get_tokenizer().vocab_size,   10_00  )






		def 						snake_case__				(							self  )       -> Any:






									"""simple docstring"""
									if not self.test_rust_tokenizer:
																return

									lowercase_	:   List[Any]  				=     self.get_tokenizer()
									lowercase_	:   List[str]  				=     self.get_rust_tokenizer()

									lowercase_	:   int  				=     """I was born in 92000, and this is falsé."""

									lowercase_	:   int  				=     tokenizer.tokenize(snake_case__  )
									lowercase_	:   str  				=     rust_tokenizer.tokenize(snake_case__  )
									self.assertListEqual(snake_case__,   snake_case__  )

									lowercase_	:   int  				=     tokenizer.encode(snake_case__,   add_special_tokens=snake_case__  )
									lowercase_	:   Union[str, Any]  				=     rust_tokenizer.encode(snake_case__,   add_special_tokens=snake_case__  )
									self.assertListEqual(snake_case__,   snake_case__  )

									lowercase_	:   str  				=     self.get_rust_tokenizer()
									lowercase_	:   Dict  				=     tokenizer.encode(snake_case__  )
									lowercase_	:   Optional[Any]  				=     rust_tokenizer.encode(snake_case__  )
									self.assertListEqual(snake_case__,   snake_case__  )






		def 						snake_case__				(							self,   snake_case__=15  )       -> int:






									"""simple docstring"""
									for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
																with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""  ):
																							lowercase_	:   Union[str, Any]  				=     self.rust_tokenizer_class.from_pretrained(snake_case__,   **snake_case__  )

																							# Simple input
																							lowercase_	:   Union[str, Any]  				=     """This is a simple input"""
																							lowercase_	:   Optional[int]  				=     ["""This is a simple input 1""", """This is a simple input 2"""]
																							lowercase_	:   Optional[Any]  				=     ("""This is a simple input""", """This is a pair""")
																							lowercase_	:   Union[str, Any]  				=     [
																							    ("""This is a simple input 1""", """This is a simple input 2"""),
																							    ("""This is a simple pair 1""", """This is a simple pair 2"""),
																							]

																							# Simple input tests
																							self.assertRaises(snake_case__,   tokenizer_r.encode,   snake_case__,   max_length=snake_case__,   padding="""max_length"""  )

																							# Simple input
																							self.assertRaises(snake_case__,   tokenizer_r.encode_plus,   snake_case__,   max_length=snake_case__,   padding="""max_length"""  )

																							# Simple input
																							self.assertRaises(
																							    snake_case__,   tokenizer_r.batch_encode_plus,   snake_case__,   max_length=snake_case__,   padding="""max_length""",   )

																							# Pair input
																							self.assertRaises(snake_case__,   tokenizer_r.encode,   snake_case__,   max_length=snake_case__,   padding="""max_length"""  )

																							# Pair input
																							self.assertRaises(snake_case__,   tokenizer_r.encode_plus,   snake_case__,   max_length=snake_case__,   padding="""max_length"""  )

																							# Pair input
																							self.assertRaises(
																							    snake_case__,   tokenizer_r.batch_encode_plus,   snake_case__,   max_length=snake_case__,   padding="""max_length""",   )






		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									pass






		def 						snake_case__				(							self  )       -> List[str]:






									"""simple docstring"""
									lowercase_	:   Tuple  				=     ReformerTokenizer(snake_case__,   keep_accents=snake_case__  )

									lowercase_	:   List[Any]  				=     tokenizer.tokenize("""This is a test"""  )
									self.assertListEqual(snake_case__,   ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(snake_case__  ),   [2_85, 46, 10, 1_70, 3_82],   )

									lowercase_	:   Tuple  				=     tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
									self.assertListEqual(
									    snake_case__,   [
									        SPIECE_UNDERLINE + """I""",
									        SPIECE_UNDERLINE + """was""",
									        SPIECE_UNDERLINE + """b""",
									        """or""",
									        """n""",
									        SPIECE_UNDERLINE + """in""",
									        SPIECE_UNDERLINE + """""",
									        """9""",
									        """2""",
									        """0""",
									        """0""",
									        """0""",
									        """,""",
									        SPIECE_UNDERLINE + """and""",
									        SPIECE_UNDERLINE + """this""",
									        SPIECE_UNDERLINE + """is""",
									        SPIECE_UNDERLINE + """f""",
									        """al""",
									        """s""",
									        """é""",
									        """.""",
									    ],   )
									lowercase_	:   List[str]  				=     tokenizer.convert_tokens_to_ids(snake_case__  )
									self.assertListEqual(
									    snake_case__,   [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4],   )

									lowercase_	:   List[Any]  				=     tokenizer.convert_ids_to_tokens(snake_case__  )
									self.assertListEqual(
									    snake_case__,   [
									        SPIECE_UNDERLINE + """I""",
									        SPIECE_UNDERLINE + """was""",
									        SPIECE_UNDERLINE + """b""",
									        """or""",
									        """n""",
									        SPIECE_UNDERLINE + """in""",
									        SPIECE_UNDERLINE + """""",
									        """<unk>""",
									        """2""",
									        """0""",
									        """0""",
									        """0""",
									        """,""",
									        SPIECE_UNDERLINE + """and""",
									        SPIECE_UNDERLINE + """this""",
									        SPIECE_UNDERLINE + """is""",
									        SPIECE_UNDERLINE + """f""",
									        """al""",
									        """s""",
									        """<unk>""",
									        """.""",
									    ],   )






		@cached_property
		def 						snake_case__				(							self  )       -> Optional[int]:






									"""simple docstring"""
									return ReformerTokenizer.from_pretrained("""google/reformer-crime-and-punishment"""  )






		@slow
		def 						snake_case__				(							self  )       -> List[str]:






									"""simple docstring"""
									lowercase_	:   str  				=     """Hello World!"""
									lowercase_	:   List[str]  				=     [1_26, 32, 2_62, 1_52, 38, 72, 2_87]

									self.assertListEqual(snake_case__,   self.big_tokenizer.encode(snake_case__  )  )






		@slow
		def 						snake_case__				(							self  )       -> str:






									"""simple docstring"""
									lowercase_	:   str  				=     (
									    """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
									    """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
									)
									lowercase_	:   str  				=     [
									    1_08,
									    2_65,
									    24,
									    1_11,
									    4,
									    2_58,
									    1_56,
									    35,
									    28,
									    2_75,
									    3,
									    2_59,
									    2_97,
									    2_60,
									    84,
									    4,
									    35,
									    1_10,
									    44,
									    8,
									    2_59,
									    91,
									    2_68,
									    21,
									    11,
									    2_09,
									    2_74,
									    1_09,
									    2_66,
									    2_77,
									    1_17,
									    86,
									    93,
									    3_15,
									    2_58,
									    2_78,
									    2_58,
									    2_77,
									    2_58,
									    0,
									    2_58,
									    2_88,
									    2_58,
									    3_19,
									    2_58,
									    0,
									    2_58,
									    0,
									    2_58,
									    0,
									    2_58,
									    0,
									    2_58,
									    2_87,
									    2_58,
									    3_15,
									    2_58,
									    2_89,
									    2_58,
									    2_78,
									    99,
									    2_69,
									    2_66,
									    2_62,
									    8,
									    2_59,
									    2_41,
									    4,
									    2_17,
									    2_30,
									    2_68,
									    2_66,
									    55,
									    1_68,
									    1_06,
									    75,
									    1_93,
									    2_66,
									    2_23,
									    27,
									    49,
									    26,
									    2_82,
									    25,
									    2_64,
									    2_99,
									    19,
									    26,
									    0,
									    2_58,
									    2_77,
									    1_17,
									    86,
									    93,
									    1_76,
									    1_83,
									    2_70,
									    11,
									    2_62,
									    42,
									    61,
									    2_65,
									]

									self.assertListEqual(snake_case__,   self.big_tokenizer.encode(snake_case__  )  )






		@require_torch
		@slow
		def 						snake_case__				(							self  )       -> Optional[Any]:






									"""simple docstring"""
									import torch

									from transformers import ReformerConfig, ReformerModel

									# Build sequence
									lowercase_	:   List[str]  				=     list(self.big_tokenizer.get_vocab().keys()  )[:10]
									lowercase_	:   Optional[Any]  				=     """ """.join(snake_case__  )
									lowercase_	:   Union[str, Any]  				=     self.big_tokenizer.encode_plus(snake_case__,   return_tensors="""pt"""  )
									lowercase_	:   Optional[int]  				=     self.big_tokenizer.batch_encode_plus([sequence, sequence],   return_tensors="""pt"""  )

									lowercase_	:   Optional[int]  				=     ReformerConfig()
									# The input gets padded during training so adjust the axial position encodings from the pretrained model value of (512, 1024)
									lowercase_	:   Optional[int]  				=     encoded_sequence["""input_ids"""].shape
									lowercase_	:   List[Any]  				=     ReformerModel(snake_case__  )

									# Reformer has config.vocab_size == tokenizer.vocab_size == len(tokenizer) - 1 = 320; len(tokenizer) is 321 (including a pad token with id 320)
									assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

									with torch.no_grad():
																model(**snake_case__  )
																model(**snake_case__  )






		@slow
		def 						snake_case__				(							self  )       -> Optional[int]:






									"""simple docstring"""
									# fmt: off
									lowercase_	:   Optional[int]  				=     {"""input_ids""": [[1_08, 2_65, 24, 1_11, 4, 2_58, 1_56, 7, 51, 2_79, 58, 7, 76, 25, 69, 2_78], [1_40, 2_43, 2_64, 1_34, 17, 2_67, 77, 2_63, 22, 2_62, 2_97, 2_58, 3_04, 1_77, 2_79, 2_66, 14, 89, 13, 35, 2_61, 2_99, 2_72, 1_37, 2_75, 2_78]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
									# fmt: on

									# This tokenizer does not know some characters like ")".
									# That is the reason why we use very simple texts here.
									# Also see https://github.com/huggingface/transformers/pull/11737#issuecomment-850769064
									lowercase_	:   Optional[Any]  				=     [
									    """This is a very simple sentence.""",
									    """The quick brown fox jumps over the lazy dog.""",
									]

									self.tokenizer_integration_test_util(
									    expected_encoding=snake_case__,   model_name="""google/reformer-crime-and-punishment""",   revision="""0e6c3decb8211d49bf881013425dc8b0448b3f5a""",   padding=snake_case__,   sequences=snake_case__,   )






def  __magic_name__  (				lowercase = 100				)				-> int:

							"""simple docstring"""



							lowercase_	:   Dict  				=     (n * (n + 1) // 2) ** 2
							lowercase_	:   List[str]  				=     n * (n + 1) * (2 * n + 1) // 6
							return sum_cubes - sum_squares


if __name__ == "__main__":
							print(F'''{solution() = }''')