






'''simple docstring'''





import unittest

from transformers import SPIECE_UNDERLINE, XLNetTokenizer, XLNetTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase						:						List[Any]  =					get_tests_dir('fixtures/test_sentencepiece.model')


@require_sentencepiece
@require_tokenizers
class  snake_case__      (					lowerCamelCase_     ,				unittest.TestCase):
					a_        =					XLNetTokenizer
					a_        =					XLNetTokenizerFast
					a_        =					True
					a_        =					True




					def 		A      (  self  :		List[Any]      )    ->							List[str]:
								super().setUp()

								# We have a SentencePiece fixture for testing
								UpperCAmelCase_      :       Tuple          =       XLNetTokenizer(snake_case__				, keep_accents=snake_case__      )
								tokenizer.sanitize_special_tokens()
								tokenizer.save_pretrained(self.tmpdirname      )




					def 		A      (  self  :		Tuple      )    ->							List[Any]:
								UpperCAmelCase_      :       Any          =       '<s>'
								UpperCAmelCase_      :       Dict          =       1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__      )				, snake_case__      )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__      )				, snake_case__      )




					def 		A      (  self  :		int      )    ->							str:
								UpperCAmelCase_      :       Dict          =       list(self.get_tokenizer().get_vocab().keys()      )

								self.assertEqual(vocab_keys[0]				, '''<unk>'''      )
								self.assertEqual(vocab_keys[1]				, '''<s>'''      )
								self.assertEqual(vocab_keys[-1]				, '''<eod>'''      )
								self.assertEqual(len(snake_case__      )				, 10_06      )




					def 		A      (  self  :		Any      )    ->							Optional[Any]:
								self.assertEqual(self.get_tokenizer().vocab_size				, 10_00      )




					def 		A      (  self  :		Optional[Any]      )    ->							Optional[Any]:
								UpperCAmelCase_      :       Tuple          =       XLNetTokenizer(snake_case__				, keep_accents=snake_case__      )

								UpperCAmelCase_      :       List[str]          =       tokenizer.tokenize('''This is a test'''      )
								self.assertListEqual(snake_case__				, ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']      )

								self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__      )				, [2_85, 46, 10, 1_70, 3_82]      )

								UpperCAmelCase_      :       Optional[int]          =       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
								self.assertListEqual(
								    snake_case__				, [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''é''',
								        '''.''',
								    ]				, )
								UpperCAmelCase_      :       Dict          =       tokenizer.convert_tokens_to_ids(snake_case__      )
								self.assertListEqual(snake_case__				, [8, 21, 84, 55, 24, 19, 7, 0, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 0, 4]      )

								UpperCAmelCase_      :       Optional[Any]          =       tokenizer.convert_ids_to_tokens(snake_case__      )
								self.assertListEqual(
								    snake_case__				, [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''<unk>''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''s''',
								        '''<unk>''',
								        '''.''',
								    ]				, )




					def 		A      (  self  :		Optional[int]      )    ->							List[str]:
								UpperCAmelCase_      :       Optional[Any]          =       XLNetTokenizer(snake_case__				, do_lower_case=snake_case__      )
								UpperCAmelCase_      :       List[Any]          =       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
								self.assertListEqual(
								    snake_case__				, [
								        SPIECE_UNDERLINE + '''''',
								        '''i''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''se''',
								        '''.''',
								    ]				, )
								self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''      )				, ['''▁he''', '''ll''', '''o''']      )




					def 		A      (  self  :		List[Any]      )    ->							List[str]:
								UpperCAmelCase_      :       List[Any]          =       XLNetTokenizer(snake_case__				, do_lower_case=snake_case__      )
								UpperCAmelCase_      :       List[Any]          =       tokenizer.tokenize('''I was born in 92000, and this is falsé.'''      )
								self.assertListEqual(
								    snake_case__				, [
								        SPIECE_UNDERLINE + '''I''',
								        SPIECE_UNDERLINE + '''was''',
								        SPIECE_UNDERLINE + '''b''',
								        '''or''',
								        '''n''',
								        SPIECE_UNDERLINE + '''in''',
								        SPIECE_UNDERLINE + '''''',
								        '''9''',
								        '''2''',
								        '''0''',
								        '''0''',
								        '''0''',
								        ''',''',
								        SPIECE_UNDERLINE + '''and''',
								        SPIECE_UNDERLINE + '''this''',
								        SPIECE_UNDERLINE + '''is''',
								        SPIECE_UNDERLINE + '''f''',
								        '''al''',
								        '''se''',
								        '''.''',
								    ]				, )




					@slow
					def 		A      (  self  :		Dict      )    ->							str:
								UpperCAmelCase_      :       Tuple          =       XLNetTokenizer.from_pretrained('''xlnet-base-cased'''      )

								UpperCAmelCase_      :       List[str]          =       tokenizer.encode('''sequence builders'''				, add_special_tokens=snake_case__      )
								UpperCAmelCase_      :       Optional[Any]          =       tokenizer.encode('''multi-sequence build'''				, add_special_tokens=snake_case__      )

								UpperCAmelCase_      :       Optional[int]          =       tokenizer.build_inputs_with_special_tokens(snake_case__      )
								UpperCAmelCase_      :       List[str]          =       tokenizer.build_inputs_with_special_tokens(snake_case__				, snake_case__      )

								assert encoded_sentence == text + [4, 3]
								assert encoded_pair == text + [4] + text_a + [4, 3]





					@slow
					def 		A      (  self  :		Tuple      )    ->							Tuple:
								# fmt: off
								UpperCAmelCase_      :       Optional[int]          =       {'input_ids': [[17, 2_14_42, 2_70, 17, 10, 1_46_45, 3_18, 34, 17, 45_46, 31_45, 7_87, 13, 77_52, 2_20_18, 23, 21, 17, 45_46, 31_45, 7_87, 13, 33_52, 1_44_31, 13, 55_00, 11, 11_76, 5_80, 13, 1_68_19, 47_97, 23, 17, 10, 1_71_35, 6_58, 19, 4_57, 79_32, 13, 1_84, 19, 31_54, 1_71_35, 64_68, 19, 14_04, 1_22_69, 19, 42_29, 53_56, 1_62_64, 46, 19, 17, 2_05_45, 1_03_95, 9, 9, 9, 11, 28, 64_21, 95_31, 2_07_29, 17, 10, 3_53, 1_70_22, 11, 21, 64_21, 95_31, 1_69_49, 17, 10, 1_15_09, 7_53, 11, 33, 95, 24_21, 73_85, 9_56, 1_44_31, 26_26, 25, 8_42, 73_85, 48_36, 21, 14_29, 22_72, 98_55, 31_20, 1_61, 2_47_38, 19, 1_32_03, 6_58, 2_18, 7_87, 21, 4_30, 1_84_82, 8_47, 26_37, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 3_22, 2_21_78, 27, 10_64, 22, 9_56, 13, 1_11_01, 14_29, 58_54, 2_43_13, 1_89_53, 40, 4_22, 2_43_66, 68, 17_58, 37, 1_04_83, 1_42_57, 31, 2_07, 2_63, 21, 2_03, 37_73, 25, 71, 97_35, 9, 4, 3], [5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 32, 20_49, 34_42, 17, 1_38_94, 33_80, 23, 95, 18, 1_76_34, 22_88, 9, 4, 3]], 'token_type_ids': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2], [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=snake_case__				, model_name='''xlnet-base-cased'''				, revision='''c841166438c31ec7ca9a106dee7bb312b73ae511'''				, )








import warnings

from ...utils import logging
from .image_processing_chinese_clip import ChineseCLIPImageProcessor


UpperCamelCase__ :   Dict						=							logging.get_logger(__name__)







class 				lowerCAmelCase_       (					lowerCamelCase_							):






				def __init__(			self      ,*snake_case__      ,**snake_case__ ):
							warnings.warn(
							    'The class ChineseCLIPFeatureExtractor is deprecated and will be removed in version 5 of Transformers.'
							    ' Please use ChineseCLIPImageProcessor instead.'      ,snake_case__      ,)
							super().__init__(*snake_case__      ,**snake_case__ )



