




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCAmelCase        =				get_tests_dir('fixtures/test_sentencepiece.model')



@require_sentencepiece
@require_tokenizers
class 							_UpperCamelCase					(	lowercase__  ,						unittest.TestCase     ):
       _UpperCamelCase				:					str         =   XLNetTokenizer
       _UpperCamelCase				:					Optional[Any]         =   XLNetTokenizerFast
       _UpperCamelCase				:					Any         =   True
       _UpperCamelCase				:					Optional[Any]         =   True

       def lowercase (		self: Optional[int]		)       -> List[str]:




              """simple docstring"""


              super().setUp()

              # We have a SentencePiece fixture for testing
              UpperCamelCase_          =  XLNetTokenizer(UpperCAmelCase__					,   keep_accents=UpperCAmelCase__		)
              tokenizer.sanitize_special_tokens()
              tokenizer.save_pretrained(self.tmpdirname		)

       def lowercase (		self: Tuple		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  '''<s>'''
              UpperCamelCase_          =  1

              self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase__		)					,   UpperCAmelCase__		)
              self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase__		)					,   UpperCAmelCase__		)

       def lowercase (		self: Tuple		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  list(self.get_tokenizer().get_vocab().keys()		)

              self.assertEqual(vocab_keys[0]					,   "<unk>"		)
              self.assertEqual(vocab_keys[1]					,   "<s>"		)
              self.assertEqual(vocab_keys[-1]					,   "<eod>"		)
              self.assertEqual(len(UpperCAmelCase__		)					,   1006		)

       def lowercase (		self: int		)       -> Union[str, Any]:




              """simple docstring"""


              self.assertEqual(self.get_tokenizer().vocab_size					,   1000		)

       def lowercase (		self: List[Any]		)       -> Tuple:




              """simple docstring"""


              UpperCamelCase_          =  XLNetTokenizer(UpperCAmelCase__					,   keep_accents=UpperCAmelCase__		)

              UpperCamelCase_          =  tokenizer.tokenize("This is a test"		)
              self.assertListEqual(UpperCAmelCase__					,   ["▁This", "▁is", "▁a", "▁t", "est"]		)

              self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)					,   [285, 46, 10, 170, 382]		)

              UpperCamelCase_          =  tokenizer.tokenize("I was born in 92000, and this is falsé."		)
              self.assertListEqual(
                  UpperCAmelCase__					,   [
                      SPIECE_UNDERLINE + "I",
                      SPIECE_UNDERLINE + "was",
                      SPIECE_UNDERLINE + "b",
                      "or",
                      "n",
                      SPIECE_UNDERLINE + "in",
                      SPIECE_UNDERLINE + "",
                      "9",
                      "2",
                      "0",
                      "0",
                      "0",
                      ",",
                      SPIECE_UNDERLINE + "and",
                      SPIECE_UNDERLINE + "this",
                      SPIECE_UNDERLINE + "is",
                      SPIECE_UNDERLINE + "f",
                      "al",
                      "s",
                      "é",
                      ".",
                  ]					,   )
              UpperCamelCase_          =  tokenizer.convert_tokens_to_ids(UpperCAmelCase__		)
              self.assertListEqual(UpperCAmelCase__					,   [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		)

              UpperCamelCase_          =  tokenizer.convert_ids_to_tokens(UpperCAmelCase__		)
              self.assertListEqual(
                  UpperCAmelCase__					,   [
                      SPIECE_UNDERLINE + "I",
                      SPIECE_UNDERLINE + "was",
                      SPIECE_UNDERLINE + "b",
                      "or",
                      "n",
                      SPIECE_UNDERLINE + "in",
                      SPIECE_UNDERLINE + "",
                      "<unk>",
                      "2",
                      "0",
                      "0",
                      "0",
                      ",",
                      SPIECE_UNDERLINE + "and",
                      SPIECE_UNDERLINE + "this",
                      SPIECE_UNDERLINE + "is",
                      SPIECE_UNDERLINE + "f",
                      "al",
                      "s",
                      "<unk>",
                      ".",
                  ]					,   )

       def lowercase (		self: Any		)       -> Any:




              """simple docstring"""


              UpperCamelCase_          =  XLNetTokenizer(UpperCAmelCase__					,   do_lower_case=UpperCAmelCase__		)
              UpperCamelCase_          =  tokenizer.tokenize("I was born in 92000, and this is falsé."		)
              self.assertListEqual(
                  UpperCAmelCase__					,   [
                      SPIECE_UNDERLINE + "",
                      "i",
                      SPIECE_UNDERLINE + "was",
                      SPIECE_UNDERLINE + "b",
                      "or",
                      "n",
                      SPIECE_UNDERLINE + "in",
                      SPIECE_UNDERLINE + "",
                      "9",
                      "2",
                      "0",
                      "0",
                      "0",
                      ",",
                      SPIECE_UNDERLINE + "and",
                      SPIECE_UNDERLINE + "this",
                      SPIECE_UNDERLINE + "is",
                      SPIECE_UNDERLINE + "f",
                      "al",
                      "se",
                      ".",
                  ]					,   )
              self.assertListEqual(tokenizer.tokenize("H\u00E9llo"		)					,   ["▁he", "ll", "o"]		)

       def lowercase (		self: int		)       -> Union[str, Any]:




              """simple docstring"""


              UpperCamelCase_          =  XLNetTokenizer(UpperCAmelCase__					,   do_lower_case=UpperCAmelCase__		)
              UpperCamelCase_          =  tokenizer.tokenize("I was born in 92000, and this is falsé."		)
              self.assertListEqual(
                  UpperCAmelCase__					,   [
                      SPIECE_UNDERLINE + "I",
                      SPIECE_UNDERLINE + "was",
                      SPIECE_UNDERLINE + "b",
                      "or",
                      "n",
                      SPIECE_UNDERLINE + "in",
                      SPIECE_UNDERLINE + "",
                      "9",
                      "2",
                      "0",
                      "0",
                      "0",
                      ",",
                      SPIECE_UNDERLINE + "and",
                      SPIECE_UNDERLINE + "this",
                      SPIECE_UNDERLINE + "is",
                      SPIECE_UNDERLINE + "f",
                      "al",
                      "se",
                      ".",
                  ]					,   )

       @slow
       def lowercase (		self: Union[str, Any]		)       -> Optional[int]:




              """simple docstring"""


              UpperCamelCase_          =  XLNetTokenizer.from_pretrained("xlnet-base-cased"		)

              UpperCamelCase_          =  tokenizer.encode("sequence builders"					,   add_special_tokens=UpperCAmelCase__		)
              UpperCamelCase_          =  tokenizer.encode("multi-sequence build"					,   add_special_tokens=UpperCAmelCase__		)

              UpperCamelCase_          =  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__		)
              UpperCamelCase_          =  tokenizer.build_inputs_with_special_tokens(UpperCAmelCase__					,   UpperCAmelCase__		)

              assert encoded_sentence == text + [4, 3]
              assert encoded_pair == text + [4] + text_a + [4, 3]



       @slow
       def lowercase (		self: Dict		)       -> Optional[Any]:




              """simple docstring"""


              UpperCamelCase_          =  {'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
              # fmt: on

              self.tokenizer_integration_test_util(
                  expected_encoding=UpperCAmelCase__					,   model_name="xlnet-base-cased"					,   revision="c841166438c31ec7ca9a106dee7bb312b73ae511"					,   )








import random
from typing import Any




def  lowerCAmelCase_ (							UpperCamelCase_  )     ->		list[Any]:
       for _ in range(len(UpperCamelCase_  )  ):
              UpperCamelCase_          =  random.randint(0					,		len(UpperCamelCase_  ) - 1  )
              UpperCamelCase_          =  random.randint(0					,		len(UpperCamelCase_  ) - 1  )
              UpperCamelCase_       , UpperCamelCase_          =  data[b], data[a]
       return data


if __name__ == "__main__":
     _UpperCAmelCase        =				[0, 1, 2, 3, 4, 5, 6, 7]
     _UpperCAmelCase        =				['python', 'says', 'hello', '!']
     print('Fisher-Yates Shuffle:')
     print('List', integers, strings)
     print('FY Shuffle', fisher_yates_shuffle(integers), fisher_yates_shuffle(strings))

