

import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__A      		=     get_tests_dir('''fixtures/test_sentencepiece.model''')



@require_sentencepiece
@require_tokenizers
class   lowercase_   (  __lowercase       ,							unittest.TestCase					):
						UpperCamelCase_      :		Optional[int]										=    XLNetTokenizer
						UpperCamelCase_      :		Dict										=    XLNetTokenizerFast
						UpperCamelCase_      :		Optional[Any]										=    True
						UpperCamelCase_      :		List[str]										=    True



						def 						UpperCamelCase_		(      self				:    Union[str, Any]		)   ->					str:
								super().setUp()

								# We have a SentencePiece fixture for testing
								_snake_case						=							XLNetTokenizer(A__      , keep_accents=A__		)
								tokenizer.sanitize_special_tokens()
								tokenizer.save_pretrained(self.tmpdirname		)



						def 						UpperCamelCase_		(      self				:    str		)   ->					Optional[Any]:
								_snake_case						=							'''<s>'''
								_snake_case						=							1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(A__		)      , A__		)
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(A__		)      , A__		)



						def 						UpperCamelCase_		(      self				:    Dict		)   ->					Any:
								_snake_case						=							list(self.get_tokenizer().get_vocab().keys()		)

								self.assertEqual(vocab_keys[0]      , '''<unk>'''		)
								self.assertEqual(vocab_keys[1]      , '''<s>'''		)
								self.assertEqual(vocab_keys[-1]      , '''<eod>'''		)
								self.assertEqual(len(A__		)      , 1006		)



						def 						UpperCamelCase_		(      self				:    Dict		)   ->					str:
								self.assertEqual(self.get_tokenizer().vocab_size      , 1000		)



						def 						UpperCamelCase_		(      self				:    Optional[Any]		)   ->					Tuple:
								_snake_case						=							XLNetTokenizer(A__      , keep_accents=A__		)

								_snake_case						=							tokenizer.tokenize('''This is a test'''		)
								self.assertListEqual(A__      , ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

								self.assertListEqual(tokenizer.convert_tokens_to_ids(A__		)      , [285, 46, 10, 170, 382]		)

								_snake_case						=							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
								self.assertListEqual(
								    A__      , [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''é''',
								        '''.''',
								    ]      , )
								_snake_case						=							tokenizer.convert_tokens_to_ids(A__		)
								self.assertListEqual(A__      , [8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]		)

								_snake_case						=							tokenizer.convert_ids_to_tokens(A__		)
								self.assertListEqual(
								    A__      , [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''<unk>''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''<unk>''',
								        '''.''',
								    ]      , )



						def 						UpperCamelCase_		(      self				:    int		)   ->					Union[str, Any]:
								_snake_case						=							XLNetTokenizer(A__      , do_lower_case=A__		)
								_snake_case						=							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
								self.assertListEqual(
								    A__      , [
								        SPIECE_UNDERLINE + '''''',
								        '''i''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''se''',
								        '''.''',
								    ]      , )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''		)      , ['''▁he''', '''ll''', '''o''']		)



						def 						UpperCamelCase_		(      self				:    List[Any]		)   ->					List[str]:
								_snake_case						=							XLNetTokenizer(A__      , do_lower_case=A__		)
								_snake_case						=							tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
								self.assertListEqual(
								    A__      , [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''se''',
								        '''.''',
								    ]      , )



						@slow
						def 						UpperCamelCase_		(      self				:    str		)   ->					Optional[Any]:
								_snake_case						=							XLNetTokenizer.from_pretrained('''xlnet-base-cased'''		)

								_snake_case						=							tokenizer.encode('''sequence builders'''      , add_special_tokens=A__		)
								_snake_case						=							tokenizer.encode('''multi-sequence build'''      , add_special_tokens=A__		)

								_snake_case						=							tokenizer.build_inputs_with_special_tokens(A__		)
								_snake_case						=							tokenizer.build_inputs_with_special_tokens(A__      , A__		)

								assert encoded_sentence == text + [4, 3]
								assert encoded_pair == text + [4] + text_a + [4, 3]







						@slow
						def 						UpperCamelCase_		(      self				:    int		)   ->					List[str]:
								# fmt: off
								_snake_case						=							{'''input_ids''': [[17, 21442, 270, 17, 10, 14645, 318, 34, 17, 4546, 3145, 787, 13, 7752, 22018, 23, 21, 17, 4546, 3145, 787, 13, 3352, 14431, 13, 5500, 11, 1176, 580, 13, 16819, 4797, 23, 17, 10, 17135, 658, 19, 457, 7932, 13, 184, 19, 3154, 17135, 6468, 19, 1404, 12269, 19, 4229, 5356, 16264, 46, 19, 17, 20545, 10395, 9, 9, 9, 11, 28, 6421, 9531, 20729, 17, 10, 353, 17022, 11, 21, 6421, 9531, 16949, 17, 10, 11509, 753, 11, 33, 95, 2421, 7385, 956, 14431, 2626, 25, 842, 7385, 4836, 21, 1429, 2272, 9855, 3120, 161, 24738, 19, 13203, 658, 218, 787, 21, 430, 18482, 847, 2637, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 322, 22178, 27, 1064, 22, 956, 13, 11101, 1429, 5854, 24313, 18953, 40, 422, 24366, 68, 1758, 37, 10483, 14257, 31, 207, 263, 21, 203, 3773, 25, 71, 9735, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 2049, 3442, 17, 13894, 3380, 23, 95, 18, 17634, 2288, 9, 4, 3]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=A__      , model_name='''xlnet-base-cased'''      , revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''      , )




from __future__ import annotations



def 				snake_case_(_UpperCamelCase			,	_UpperCamelCase					)							->     bool:



  """simple docstring"""






  _snake_case						=							get_failure_array(_UpperCamelCase					)

  # 2) Step through text searching for pattern
  _snake_case,					_snake_case						=							0, 0  # index into text, pattern
  while i < len(_UpperCamelCase					):
    if pattern[j] == text[i]:
      if j == (len(_UpperCamelCase					) - 1):
        return True
      j += 1

    # if this is a prefix in our pattern
    # just go back far enough to continue
    elif j > 0:
      _snake_case						=							failure[j - 1]
      continue
    i += 1
  return False



def 				snake_case_(_UpperCamelCase					)							->     list[int]:



  """simple docstring"""






  _snake_case						=							[0]
  _snake_case						=							0
  _snake_case						=							1
  while j < len(_UpperCamelCase					):
    if pattern[i] == pattern[j]:
      i += 1
    elif i > 0:
      _snake_case						=							failure[i - 1]
      continue
    j += 1
    failure.append(_UpperCamelCase					)
  return failure


if __name__ == "__main__":
 # Test 1)
 __A      		=     '''abc1abc12'''
 __A      		=     '''alskfjaldsabc1abc1abc12k23adsfabcabc'''
 __A      		=     '''alskfjaldsk23adsfabcabc'''
 assert kmp(pattern, texta) and not kmp(pattern, texta)

 # Test 2)
 __A      		=     '''ABABX'''
 __A      		=     '''ABABZABABYABABX'''
 assert kmp(pattern, text)

 # Test 3)
 __A      		=     '''AAAB'''
 __A      		=     '''ABAAAAAB'''
 assert kmp(pattern, text)

 # Test 4)
 __A      		=     '''abcdabcy'''
 __A      		=     '''abcxabcdabxabcdabcdabcy'''
 assert kmp(pattern, text)

 # Test 5)
 __A      		=     '''aabaabaaa'''
 assert get_failure_array(pattern) == [0, 1, 0, 1, 2, 3, 4, 5, 2]

