






'''simple docstring'''
import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__magic_name__		:     str					=    get_tests_dir('''fixtures/test_sentencepiece.model''')







@require_sentencepiece
@require_tokenizers
class 	UpperCamelCase__					(     a_      ,     unittest.TestCase			):

		"""simple docstring"""
		UpperCAmelCase__     			=						XLNetTokenizer
		UpperCAmelCase__     			=						XLNetTokenizerFast
		UpperCAmelCase__     			=						True
		UpperCAmelCase__     			=						True




		def       snake_case     ( self						:						List[Any]    ):







							"""simple docstring"""
							super().setUp()

							# We have a SentencePiece fixture for testing
							_lowercase			= XLNetTokenizer(_a							,    keep_accents=_a    )
							tokenizer.sanitize_special_tokens()
							tokenizer.save_pretrained(self.tmpdirname    )




		def       snake_case     ( self						:						Dict    ):







							"""simple docstring"""
							_lowercase			= "<s>"
							_lowercase			= 1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a    )							,    _a    )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a    )							,    _a    )




		def       snake_case     ( self						:						Union[str, Any]    ):







							"""simple docstring"""
							_lowercase			= list(self.get_tokenizer().get_vocab().keys()    )

							self.assertEqual(vocab_keys[0]							,    "<unk>"    )
							self.assertEqual(vocab_keys[1]							,    "<s>"    )
							self.assertEqual(vocab_keys[-1]							,    "<eod>"    )
							self.assertEqual(len(_a    )							,    1_0_0_6    )




		def       snake_case     ( self						:						Any    ):







							"""simple docstring"""
							self.assertEqual(self.get_tokenizer().vocab_size							,    1_0_0_0    )




		def       snake_case     ( self						:						Dict    ):







							"""simple docstring"""
							_lowercase			= XLNetTokenizer(_a							,    keep_accents=_a    )

							_lowercase			= tokenizer.tokenize("This is a test"    )
							self.assertListEqual(_a							,    ["▁This", "▁is", "▁a", "▁t", "est"]    )

							self.assertListEqual(tokenizer.convert_tokens_to_ids(_a    )							,    [2_8_5, 4_6, 1_0, 1_7_0, 3_8_2]    )

							_lowercase			= tokenizer.tokenize("I was born in 92000, and this is falsé."    )
							self.assertListEqual(
							    _a							,    [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "é",
							        ".",
							    ]							,    )
							_lowercase			= tokenizer.convert_tokens_to_ids(_a    )
							self.assertListEqual(_a							,    [8, 2_1, 8_4, 5_5, 2_4, 1_9, 7, 0, 6_0_2, 3_4_7, 3_4_7, 3_4_7, 3, 1_2, 6_6, 4_6, 7_2, 8_0, 6, 0, 4]    )

							_lowercase			= tokenizer.convert_ids_to_tokens(_a    )
							self.assertListEqual(
							    _a							,    [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "<unk>",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "s",
							        "<unk>",
							        ".",
							    ]							,    )




		def       snake_case     ( self						:						Union[str, Any]    ):







							"""simple docstring"""
							_lowercase			= XLNetTokenizer(_a							,    do_lower_case=_a    )
							_lowercase			= tokenizer.tokenize("I was born in 92000, and this is falsé."    )
							self.assertListEqual(
							    _a							,    [
							        SPIECE_UNDERLINE + "",
							        "i",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "se",
							        ".",
							    ]							,    )
							self.assertListEqual(tokenizer.tokenize("H\u00E9llo"    )							,    ["▁he", "ll", "o"]    )




		def       snake_case     ( self						:						List[Any]    ):







							"""simple docstring"""
							_lowercase			= XLNetTokenizer(_a							,    do_lower_case=_a    )
							_lowercase			= tokenizer.tokenize("I was born in 92000, and this is falsé."    )
							self.assertListEqual(
							    _a							,    [
							        SPIECE_UNDERLINE + "I",
							        SPIECE_UNDERLINE + "was",
							        SPIECE_UNDERLINE + "b",
							        "or",
							        "n",
							        SPIECE_UNDERLINE + "in",
							        SPIECE_UNDERLINE + "",
							        "9",
							        "2",
							        "0",
							        "0",
							        "0",
							        ",",
							        SPIECE_UNDERLINE + "and",
							        SPIECE_UNDERLINE + "this",
							        SPIECE_UNDERLINE + "is",
							        SPIECE_UNDERLINE + "f",
							        "al",
							        "se",
							        ".",
							    ]							,    )




		@slow
		def       snake_case     ( self						:						List[str]    ):







							"""simple docstring"""
							_lowercase			= XLNetTokenizer.from_pretrained("xlnet-base-cased"    )

							_lowercase			= tokenizer.encode("sequence builders"							,    add_special_tokens=_a    )
							_lowercase			= tokenizer.encode("multi-sequence build"							,    add_special_tokens=_a    )

							_lowercase			= tokenizer.build_inputs_with_special_tokens(_a    )
							_lowercase			= tokenizer.build_inputs_with_special_tokens(_a							,    _a    )

							assert encoded_sentence == text + [4, 3]
							assert encoded_pair == text + [4] + text_a + [4, 3]




		@slow
		def       snake_case     ( self						:						Any    ):







							"""simple docstring"""
							# fmt: off
							_lowercase			= {"input_ids": [[1_7, 2_1_4_4_2, 2_7_0, 1_7, 1_0, 1_4_6_4_5, 3_1_8, 3_4, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 7_7_5_2, 2_2_0_1_8, 2_3, 2_1, 1_7, 4_5_4_6, 3_1_4_5, 7_8_7, 1_3, 3_3_5_2, 1_4_4_3_1, 1_3, 5_5_0_0, 1_1, 1_1_7_6, 5_8_0, 1_3, 1_6_8_1_9, 4_7_9_7, 2_3, 1_7, 1_0, 1_7_1_3_5, 6_5_8, 1_9, 4_5_7, 7_9_3_2, 1_3, 1_8_4, 1_9, 3_1_5_4, 1_7_1_3_5, 6_4_6_8, 1_9, 1_4_0_4, 1_2_2_6_9, 1_9, 4_2_2_9, 5_3_5_6, 1_6_2_6_4, 4_6, 1_9, 1_7, 2_0_5_4_5, 1_0_3_9_5, 9, 9, 9, 1_1, 2_8, 6_4_2_1, 9_5_3_1, 2_0_7_2_9, 1_7, 1_0, 3_5_3, 1_7_0_2_2, 1_1, 2_1, 6_4_2_1, 9_5_3_1, 1_6_9_4_9, 1_7, 1_0, 1_1_5_0_9, 7_5_3, 1_1, 3_3, 9_5, 2_4_2_1, 7_3_8_5, 9_5_6, 1_4_4_3_1, 2_6_2_6, 2_5, 8_4_2, 7_3_8_5, 4_8_3_6, 2_1, 1_4_2_9, 2_2_7_2, 9_8_5_5, 3_1_2_0, 1_6_1, 2_4_7_3_8, 1_9, 1_3_2_0_3, 6_5_8, 2_1_8, 7_8_7, 2_1, 4_3_0, 1_8_4_8_2, 8_4_7, 2_6_3_7, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2_2, 2_2_1_7_8, 2_7, 1_0_6_4, 2_2, 9_5_6, 1_3, 1_1_1_0_1, 1_4_2_9, 5_8_5_4, 2_4_3_1_3, 1_8_9_5_3, 4_0, 4_2_2, 2_4_3_6_6, 6_8, 1_7_5_8, 3_7, 1_0_4_8_3, 1_4_2_5_7, 3_1, 2_0_7, 2_6_3, 2_1, 2_0_3, 3_7_7_3, 2_5, 7_1, 9_7_3_5, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_2, 2_0_4_9, 3_4_4_2, 1_7, 1_3_8_9_4, 3_3_8_0, 2_3, 9_5, 1_8, 1_7_6_3_4, 2_2_8_8, 9, 4, 3]], "token_type_ids": [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=_a							,    model_name="xlnet-base-cased"							,    revision="c841166438c31ec7ca9a106dee7bb312b73ae511"							,    )



"""simple docstring"""




import builtins
import sys

from ...utils.imports import _is_package_available
from . import cursor, input
from .helpers import Direction, clear_line, forceWrite, linebreak, move_cursor, reset_cursor, writeColor
from .keymap import KEYMAP


_A     :			str  						=			False
try:
   _A     :			Dict  						=			_is_package_available("""google.colab""")
except ModuleNotFoundError:
   pass

@input.register
class 						a__		:
   def __init__(     self      ,       _a = None      ,       _a = []							):
          lowercase       :							Union[str, Any]									=				0
          lowercase       :							str									=				choices
          lowercase       :							List[Any]									=				prompt
          if sys.platform == "win32":
                 lowercase       :							Union[str, Any]									=				"*"
          else:
                 lowercase       :							int									=				"➔ "
   def      __magic_name__     (     self      ,       _a      ,       _a = ""							):
          if sys.platform != "win32":
                 writeColor(self.choices[index]      ,       32      ,       _a							)
          else:
                 forceWrite(self.choices[index]      ,       _a							)
   def      __magic_name__     (     self      ,       _a							):
          if index == self.position:
                 forceWrite(f""" {self.arrow_char} """							)
                 self.write_choice(_a							)
          else:
                 forceWrite(f"""    {self.choices[index]}"""							)
          reset_cursor()
   def      __magic_name__     (     self      ,       _a      ,       _a = 1							):
          lowercase       :							Optional[int]									=				self.position
          if direction == Direction.DOWN:
                 if self.position + 1 >= len(self.choices							):
                        return
                 self.position += num_spaces
          else:
                 if self.position - 1 < 0:
                        return
                 self.position -= num_spaces
          clear_line()
          self.print_choice(_a							)
          move_cursor(_a      ,       direction.name							)
          self.print_choice(self.position							)
   @input.mark(KEYMAP["up"]							)
   def      __magic_name__     (     self							):
          self.move_direction(Direction.UP							)
   @input.mark(KEYMAP["down"]							)
   def      __magic_name__     (     self							):
          self.move_direction(Direction.DOWN							)
   @input.mark(KEYMAP["newline"]							)
   def      __magic_name__     (     self							):
          move_cursor(len(self.choices							) - self.position      ,       "DOWN"							)
          return self.position
   @input.mark(KEYMAP["interrupt"]							)
   def      __magic_name__     (     self							):
          move_cursor(len(self.choices							) - self.position      ,       "DOWN"							)
          raise KeyboardInterrupt
   @input.mark_multiple(*[KEYMAP[str(_a							)] for number in range(10							)]							)
   def      __magic_name__     (     self							):
          lowercase       :							List[str]									=				int(chr(self.current_selection							)							)
          lowercase       :							Union[str, Any]									=				index - self.position
          if index == self.position:
                 return
          if index < len(self.choices							):
                 if self.position > index:
                        self.move_direction(Direction.UP      ,       -movement							)
                 elif self.position < index:
                        self.move_direction(Direction.DOWN      ,       _a							)
                 else:
                        return
          else:
                 return

   def      __magic_name__     (     self      ,       _a = 0							):
          if self.prompt:
                 linebreak()
                 forceWrite(self.prompt      ,       "\n"							)
                 if in_colab:
                        forceWrite("Please input a choice index (starting from 0), and press enter"      ,       "\n"							)
                 else:
                        forceWrite("Please select a choice using the arrow or number keys, and selecting with enter"      ,       "\n"							)
          lowercase       :							Tuple									=				default_choice
          for i in range(len(self.choices							)							):
                 self.print_choice(_a							)
                 forceWrite("\n"							)
          move_cursor(len(self.choices							) - self.position      ,       "UP"							)
          with cursor.hide():
                 while True:
                        if in_colab:
                               try:
                                      lowercase       :							Tuple									=				int(builtins.input()							)
                               except ValueError:
                                      lowercase       :							Optional[int]									=				default_choice
                        else:
                               lowercase       :							int									=				self.handle_input()
                        if choice is not None:
                               reset_cursor()
                               for _ in range(len(self.choices							) + 1							):
                                      move_cursor(1      ,       "UP"							)
                                      clear_line()
                               self.write_choice(_a      ,       "\n"							)
                               return choice


