




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase								=	get_tests_dir('fixtures/test_sentencepiece.model')





@require_sentencepiece
@require_tokenizers
class 					__lowerCamelCase			(       UpperCamelCase__      ,							unittest.TestCase       ):




    """simple docstring"""

    snake_case__   = XLNetTokenizer
    snake_case__   = XLNetTokenizerFast
    snake_case__   = True
    snake_case__   = True


    def   a (			self	:	str  ) ->			str:
         super().setUp()

         # We have a SentencePiece fixture for testing
         lowerCAmelCase__            =					XLNetTokenizer(SCREAMING_SNAKE_CASE__  ,      keep_accents=SCREAMING_SNAKE_CASE__  )
         tokenizer.sanitize_special_tokens()
         tokenizer.save_pretrained(self.tmpdirname  )


    def   a (			self	:	List[str]  ) ->			List[Any]:
         lowerCAmelCase__            =					"<s>"
         lowerCAmelCase__            =					1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(SCREAMING_SNAKE_CASE__  )  ,      SCREAMING_SNAKE_CASE__  )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(SCREAMING_SNAKE_CASE__  )  ,      SCREAMING_SNAKE_CASE__  )


    def   a (			self	:	Union[str, Any]  ) ->			str:
         lowerCAmelCase__            =					list(self.get_tokenizer().get_vocab().keys()  )

         self.assertEqual(vocab_keys[0]  ,      "<unk>"  )
         self.assertEqual(vocab_keys[1]  ,      "<s>"  )
         self.assertEqual(vocab_keys[-1]  ,      "<eod>"  )
         self.assertEqual(len(SCREAMING_SNAKE_CASE__  )  ,      1_006  )


    def   a (			self	:	int  ) ->			Dict:
         self.assertEqual(self.get_tokenizer().vocab_size  ,      1_000  )


    def   a (			self	:	List[str]  ) ->			Any:
         lowerCAmelCase__            =					XLNetTokenizer(SCREAMING_SNAKE_CASE__  ,      keep_accents=SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					tokenizer.tokenize("This is a test"  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      ["▁This", "▁is", "▁a", "▁t", "est"]  )

         self.assertListEqual(tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )  ,      [285, 46, 10, 170, 382]  )

         lowerCAmelCase__            =					tokenizer.tokenize("I was born in 92000, and this is falsé."  )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__  ,      [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "é",
                 ".",
             ]  ,      )
         lowerCAmelCase__            =					tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(SCREAMING_SNAKE_CASE__  ,      [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]  )

         lowerCAmelCase__            =					tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE__  )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__  ,      [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "<unk>",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "s",
                 "<unk>",
                 ".",
             ]  ,      )


    def   a (			self	:	Optional[int]  ) ->			Optional[Any]:
         lowerCAmelCase__            =					XLNetTokenizer(SCREAMING_SNAKE_CASE__  ,      do_lower_case=SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					tokenizer.tokenize("I was born in 92000, and this is falsé."  )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__  ,      [
                 SPIECE_UNDERLINE + "",
                 "i",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "se",
                 ".",
             ]  ,      )
         self.assertListEqual(tokenizer.tokenize("H\u00E9llo"  )  ,      ["▁he", "ll", "o"]  )


    def   a (			self	:	List[Any]  ) ->			Optional[int]:
         lowerCAmelCase__            =					XLNetTokenizer(SCREAMING_SNAKE_CASE__  ,      do_lower_case=SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					tokenizer.tokenize("I was born in 92000, and this is falsé."  )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE__  ,      [
                 SPIECE_UNDERLINE + "I",
                 SPIECE_UNDERLINE + "was",
                 SPIECE_UNDERLINE + "b",
                 "or",
                 "n",
                 SPIECE_UNDERLINE + "in",
                 SPIECE_UNDERLINE + "",
                 "9",
                 "2",
                 "0",
                 "0",
                 "0",
                 ",",
                 SPIECE_UNDERLINE + "and",
                 SPIECE_UNDERLINE + "this",
                 SPIECE_UNDERLINE + "is",
                 SPIECE_UNDERLINE + "f",
                 "al",
                 "se",
                 ".",
             ]  ,      )


    @slow
    def   a (			self	:	Any  ) ->			Any:
         lowerCAmelCase__            =					XLNetTokenizer.from_pretrained("xlnet-base-cased"  )

         lowerCAmelCase__            =					tokenizer.encode("sequence builders"  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					tokenizer.encode("multi-sequence build"  ,      add_special_tokens=SCREAMING_SNAKE_CASE__  )

         lowerCAmelCase__            =					tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__  )
         lowerCAmelCase__            =					tokenizer.build_inputs_with_special_tokens(SCREAMING_SNAKE_CASE__  ,      SCREAMING_SNAKE_CASE__  )

         assert encoded_sentence == text + [4, 3]
         assert encoded_pair == text + [4] + text_a + [4, 3]


    @slow
    def   a (			self	:	Union[str, Any]  ) ->			Any:
         # fmt: off
         lowerCAmelCase__            =					{"input_ids": [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=SCREAMING_SNAKE_CASE__  ,      model_name="xlnet-base-cased"  ,      revision="c841166438c31ec7ca9a106dee7bb312b73ae511"  ,      )






from typing import List

from .keymap import KEYMAP, get_character




def 		lowerCAmelCase_ (						lowerCamelCase ):

				def decorator(lowerCamelCase ):
								__magic_name__						:						str										=getattr(lowerCamelCase						,  """handle_key"""						,  [] )
								handle += [key]
								setattr(lowerCamelCase						,  """handle_key"""						,  lowerCamelCase )
								return func

				return decorator




def 		lowerCAmelCase_ (						*lowerCamelCase ):

				def decorator(lowerCamelCase ):
								__magic_name__						:						Dict										=getattr(lowerCamelCase						,  """handle_key"""						,  [] )
								handle += keys
								setattr(lowerCamelCase						,  """handle_key"""						,  lowerCamelCase )
								return func

				return decorator




class 		__A   (      UpperCamelCase__      ):

						def __new__(		cls :Dict     ,			__snake_case :Optional[Any]     ,			__snake_case :Union[str, Any]     ,			__snake_case :List[str]			):
										'''simple docstring'''

										__magic_name__						:						int										=super().__new__(cls     ,			__snake_case     ,			__snake_case     ,			__snake_case			)
										if not hasattr(__snake_case     ,			"""key_handler"""			):
														setattr(__snake_case     ,			"""key_handler"""     ,			{}			)
										setattr(__snake_case     ,			"""handle_input"""     ,			KeyHandler.handle_input			)

										for value in attrs.values():
														__magic_name__						:						int										=getattr(__snake_case     ,			"""handle_key"""     ,			[]			)
														for key in handled_keys:
																		__magic_name__						:						List[str]										=value
										return new_cls


						@staticmethod
						def      A__    (		cls :Optional[int]			):
										'''simple docstring'''

										__magic_name__						:						Union[str, Any]										=get_character()
										if char != KEYMAP["undefined"]:
														__magic_name__						:						Optional[int]										=ord(__snake_case			)
										__magic_name__						:						int										=cls.key_handler.get(__snake_case			)
										if handler:
														__magic_name__						:						Dict										=char
														return handler(cls			)
										else:
														return None




def 		lowerCAmelCase_ (						cls ):
				return KeyHandler(cls.__name__						,  cls.__bases__						,  cls.__dict__.copy() )




