





"""simple docstring"""




import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowercase_          =		get_tests_dir("""fixtures/test_sentencepiece.model""")
@require_sentencepiece
@require_tokenizers
class     SCREAMING_SNAKE_CASE       (UpperCAmelCase      ,     unittest.TestCase				):
   _UpperCamelCase   :		Optional[Any]			 =	XLNetTokenizer
   _UpperCamelCase   :		Dict			 =	XLNetTokenizerFast
   _UpperCamelCase   :		Any			 =	True
   _UpperCamelCase   :		Any			 =	True


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]	)->       Any:





     """simple docstring"""







     super().setUp()

     # We have a SentencePiece fixture for testing
     lowercase__           =  XLNetTokenizer(a      ,					keep_accents=a	)
     tokenizer.sanitize_special_tokens()
     tokenizer.save_pretrained(self.tmpdirname	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]	)->       Optional[int]:





     """simple docstring"""







     lowercase__           =  '<s>'
     lowercase__           =  1

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(a	)      ,					a	)
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(a	)      ,					a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any	)->       List[Any]:





     """simple docstring"""







     lowercase__           =  list(self.get_tokenizer().get_vocab().keys()	)

     self.assertEqual(vocab_keys[0]      ,					'<unk>'	)
     self.assertEqual(vocab_keys[1]      ,					'<s>'	)
     self.assertEqual(vocab_keys[-1]      ,					'<eod>'	)
     self.assertEqual(len(a	)      ,					1_006	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       int:





     """simple docstring"""







     self.assertEqual(self.get_tokenizer().vocab_size      ,					1_000	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       Optional[Any]:





     """simple docstring"""







     lowercase__           =  XLNetTokenizer(a      ,					keep_accents=a	)

     lowercase__           =  tokenizer.tokenize('This is a test'	)
     self.assertListEqual(a      ,					['▁This', '▁is', '▁a', '▁t', 'est']	)

     self.assertListEqual(tokenizer.convert_tokens_to_ids(a	)      ,					[285, 46, 10, 170, 382]	)

     lowercase__           =  tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
     self.assertListEqual(
         a      ,					[
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             'é',
             '.',
         ]      ,					)
     lowercase__           =  tokenizer.convert_tokens_to_ids(a	)
     self.assertListEqual(a      ,					[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]	)

     lowercase__           =  tokenizer.convert_ids_to_tokens(a	)
     self.assertListEqual(
         a      ,					[
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '<unk>',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             's',
             '<unk>',
             '.',
         ]      ,					)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any	)->       Union[str, Any]:





     """simple docstring"""







     lowercase__           =  XLNetTokenizer(a      ,					do_lower_case=a	)
     lowercase__           =  tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
     self.assertListEqual(
         a      ,					[
             SPIECE_UNDERLINE + '',
             'i',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             'se',
             '.',
         ]      ,					)
     self.assertListEqual(tokenizer.tokenize('H\u00E9llo'	)      ,					['▁he', 'll', 'o']	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Dict	)->       List[Any]:





     """simple docstring"""







     lowercase__           =  XLNetTokenizer(a      ,					do_lower_case=a	)
     lowercase__           =  tokenizer.tokenize('I was born in 92000, and this is falsé.'	)
     self.assertListEqual(
         a      ,					[
             SPIECE_UNDERLINE + 'I',
             SPIECE_UNDERLINE + 'was',
             SPIECE_UNDERLINE + 'b',
             'or',
             'n',
             SPIECE_UNDERLINE + 'in',
             SPIECE_UNDERLINE + '',
             '9',
             '2',
             '0',
             '0',
             '0',
             ',',
             SPIECE_UNDERLINE + 'and',
             SPIECE_UNDERLINE + 'this',
             SPIECE_UNDERLINE + 'is',
             SPIECE_UNDERLINE + 'f',
             'al',
             'se',
             '.',
         ]      ,					)


   @slow
   def 	SCREAMING_SNAKE_CASE_							(   self      :    Union[str, Any]	)->       Any:





     """simple docstring"""







     lowercase__           =  XLNetTokenizer.from_pretrained('xlnet-base-cased'	)

     lowercase__           =  tokenizer.encode('sequence builders'      ,					add_special_tokens=a	)
     lowercase__           =  tokenizer.encode('multi-sequence build'      ,					add_special_tokens=a	)

     lowercase__           =  tokenizer.build_inputs_with_special_tokens(a	)
     lowercase__           =  tokenizer.build_inputs_with_special_tokens(a      ,					a	)

     assert encoded_sentence == text + [4, 3]
     assert encoded_pair == text + [4] + text_a + [4, 3]


   @slow
   def 	SCREAMING_SNAKE_CASE_							(   self      :    int	)->       Dict:





     """simple docstring"""







     lowercase__           =  {'input_ids': [[17, 21_442, 270, 17, 10, 14_645, 318, 34, 17, 4_546, 3_145, 787, 13, 7_752, 22_018, 23, 21, 17, 4_546, 3_145, 787, 13, 3_352, 14_431, 13, 5_500, 11, 1_176, 580, 13, 16_819, 4_797, 23, 17, 10, 17_135, 658, 19, 457, 7_932, 13, 184, 19, 3_154, 17_135, 6_468, 19, 1_404, 12_269, 19, 4_229, 5_356, 16_264, 46, 19, 17, 20_545, 10_395, 9, 9, 9, 11, 28, 6_421, 9_531, 20_729, 17, 10, 353, 17_022, 11, 21, 6_421, 9_531, 16_949, 17, 10, 11_509, 753, 11, 33, 95, 2_421, 7_385, 956, 14_431, 2_626, 25, 842, 7_385, 4_836, 21, 1_429, 2_272, 9_855, 3_120, 161, 24_738, 19, 13_203, 658, 218, 787, 21, 430, 18_482, 847, 2_637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22_178, 27, 1_064, 22, 956, 13, 11_101, 1_429, 5_854, 24_313, 18_953, 40, 422, 24_366, 68, 1_758, 37, 10_483, 14_257, 31, 207, 263, 21, 203, 3_773, 25, 71, 9_735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2_049, 3_442, 17, 13_894, 3_380, 23, 95, 18, 17_634, 2_288, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=a      ,					model_name='xlnet-base-cased'      ,					revision='c841166438c31ec7ca9a106dee7bb312b73ae511'      ,					)









from collections.abc import Iterator, MutableMapping
from dataclasses import dataclass
from typing import Generic, TypeVar

lowercase_          =		TypeVar("""KEY""")
lowercase_          =		TypeVar("""VAL""")
@dataclass(frozen=UpperCAmelCase      ,     slots=UpperCAmelCase				)
class     SCREAMING_SNAKE_CASE       (Generic[KEY, VAL]				):
   _UpperCamelCase   :		KEY
   _UpperCamelCase   :		VAL






class     SCREAMING_SNAKE_CASE       (_Item				):


   def __init__(   self      :    Optional[int]	)->       None:





     """simple docstring"""







     super().__init__(a      ,					a	)


   def __bool__(   self      :    str	)->       bool:





     """simple docstring"""







     return False






lowercase_          =		_DeletedItem()
class     SCREAMING_SNAKE_CASE       (MutableMapping[KEY, VAL]				):


   def __init__(   self      :    Tuple      ,					a      :    int = 8      ,					a      :    float = 0.75	)->       None:





     """simple docstring"""







     lowercase__           =  initial_block_size
     lowercase__           =  [None] * initial_block_size
     assert 0.0 < capacity_factor < 1.0
     lowercase__           =  capacity_factor
     lowercase__           =  0


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    KEY	)->       int:





     """simple docstring"""







     return hash(a	) % len(self._buckets	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[Any]      ,					a      :    int	)->       int:





     """simple docstring"""







     return (ind + 1) % len(self._buckets	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    int      ,					a      :    KEY      ,					a      :    VAL	)->       bool:





     """simple docstring"""







     lowercase__           =  self._buckets[ind]
     if not stored:
       lowercase__           =  _Item(a      ,					a	)
       self._len += 1
       return True
     elif stored.key == key:
       lowercase__           =  _Item(a      ,					a	)
       return True
     else:
       return False


   def 	SCREAMING_SNAKE_CASE_							(   self      :    str	)->       bool:





     """simple docstring"""







     lowercase__           =  len(self._buckets	) * self._capacity_factor
     return len(self	) >= int(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple	)->       bool:





     """simple docstring"""







     if len(self._buckets	) <= self._initial_block_size:
       return False
     lowercase__           =  len(self._buckets	) * self._capacity_factor / 2
     return len(self	) < limit


   def 	SCREAMING_SNAKE_CASE_							(   self      :    List[str]      ,					a      :    int	)->       None:





     """simple docstring"""







     lowercase__           =  self._buckets
     lowercase__           =  [None] * new_size
     lowercase__           =  0
     for item in old_buckets:
       if item:
         self._add_item(item.key      ,					item.val	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]	)->       None:





     """simple docstring"""







     self._resize(len(self._buckets	) * 2	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Optional[int]	)->       None:





     """simple docstring"""







     self._resize(len(self._buckets	) // 2	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Any      ,					a      :    KEY	)->       Iterator[int]:





     """simple docstring"""







     lowercase__           =  self._get_bucket_index(a	)
     for _ in range(len(self._buckets	)	):
       yield ind
       lowercase__           =  self._get_next_ind(a	)


   def 	SCREAMING_SNAKE_CASE_							(   self      :    Tuple      ,					a      :    KEY      ,					a      :    VAL	)->       None:





     """simple docstring"""







     for ind in self._iterate_buckets(a	):
       if self._try_set(a      ,					a      ,					a	):
         break


   def __setitem__(   self      :    List[Any]      ,					a      :    KEY      ,					a      :    VAL	)->       None:





     """simple docstring"""







     if self._is_full():
       self._size_up()

     self._add_item(a      ,					a	)


   def __delitem__(   self      :    str      ,					a      :    KEY	)->       None:





     """simple docstring"""







     for ind in self._iterate_buckets(a	):
       lowercase__           =  self._buckets[ind]
       if item is None:
         raise KeyError(a	)
       if item is _deleted:
         continue
       if item.key == key:
         lowercase__           =  _deleted
         self._len -= 1
         break
     if self._is_sparse():
       self._size_down()


   def __getitem__(   self      :    List[str]      ,					a      :    KEY	)->       VAL:





     """simple docstring"""







     for ind in self._iterate_buckets(a	):
       lowercase__           =  self._buckets[ind]
       if item is None:
         break
       if item is _deleted:
         continue
       if item.key == key:
         return item.val
     raise KeyError(a	)


   def __len__(   self      :    Tuple	)->       int:





     """simple docstring"""







     return self._len


   def __iter__(   self      :    int	)->       Iterator[KEY]:





     """simple docstring"""







     yield from (item.key for item in self._buckets if item)


   def __repr__(   self      :    Union[str, Any]	)->       str:





     """simple docstring"""







     lowercase__           =  ' ,'.join(
         f"""{item.key}: {item.val}""" for item in self._buckets if item	)
     return f"""HashMap({val_string})"""


