


'''simple docstring'''
import os
import unittest

from transformers import BertTokenizerFast
from transformers.models.bert.tokenization_bert import (
    VOCAB_FILES_NAMES,
    BasicTokenizer,
    BertTokenizer,
    WordpieceTokenizer,
    _is_control,
    _is_punctuation,
    _is_whitespace,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin, filter_non_english
@require_tokenizers
class   lowercase      (				SCREAMING_SNAKE_CASE__      ,						unittest.TestCase   ):

		"""simple docstring"""

		_a       					=							BertTokenizer
		_a       					=							BertTokenizerFast
		_a       					=							True
		_a       					=							True
		_a       					=							filter_non_english




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							super().setUp()

							UpperCamelCase__  :Optional[Any]				   =					[
							    '''[UNK]''',
							    '''[CLS]''',
							    '''[SEP]''',
							    '''[PAD]''',
							    '''[MASK]''',
							    '''want''',
							    '''##want''',
							    '''##ed''',
							    '''wa''',
							    '''un''',
							    '''runn''',
							    '''##ing''',
							    ''',''',
							    '''low''',
							    '''lowest''',
							]
							UpperCamelCase__  :List[str]				   =					os.path.join(self.tmpdirname       ,  VOCAB_FILES_NAMES['''vocab_file''']				)
							with open(self.vocab_file       ,  '''w'''       ,  encoding='''utf-8'''				) as vocab_writer:
												vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]				)				)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :Any				   =					'''UNwant\u00E9d,running'''
							UpperCamelCase__  :str				   =					'''unwanted, running'''
							return input_text, output_text




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[Any]				   =					self.tokenizer_class(self.vocab_file				)

							UpperCamelCase__  :Union[str, Any]				   =					tokenizer.tokenize('''UNwant\u00E9d,running'''				)
							self.assertListEqual(__lowerCamelCase       ,  ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']				)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase				)       ,  [9, 6, 7, 12, 10, 11]				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							if not self.test_rust_tokenizer:
												return

							UpperCamelCase__  :List[str]				   =					self.get_tokenizer()
							UpperCamelCase__  :List[str]				   =					self.get_rust_tokenizer()

							UpperCamelCase__  :Optional[int]				   =					'''UNwant\u00E9d,running'''

							UpperCamelCase__  :Union[str, Any]				   =					tokenizer.tokenize(__lowerCamelCase				)
							UpperCamelCase__  :Optional[int]				   =					rust_tokenizer.tokenize(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)

							UpperCamelCase__  :Tuple				   =					tokenizer.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)
							UpperCamelCase__  :Dict				   =					rust_tokenizer.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)

							UpperCamelCase__  :Dict				   =					self.get_rust_tokenizer()
							UpperCamelCase__  :Optional[Any]				   =					tokenizer.encode(__lowerCamelCase				)
							UpperCamelCase__  :int				   =					rust_tokenizer.encode(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)

							# With lower casing
							UpperCamelCase__  :Optional[int]				   =					self.get_tokenizer(do_lower_case=__lowerCamelCase				)
							UpperCamelCase__  :Tuple				   =					self.get_rust_tokenizer(do_lower_case=__lowerCamelCase				)

							UpperCamelCase__  :Any				   =					'''UNwant\u00E9d,running'''

							UpperCamelCase__  :List[Any]				   =					tokenizer.tokenize(__lowerCamelCase				)
							UpperCamelCase__  :Any				   =					rust_tokenizer.tokenize(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)

							UpperCamelCase__  :int				   =					tokenizer.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)
							UpperCamelCase__  :List[str]				   =					rust_tokenizer.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)

							UpperCamelCase__  :Any				   =					self.get_rust_tokenizer()
							UpperCamelCase__  :int				   =					tokenizer.encode(__lowerCamelCase				)
							UpperCamelCase__  :str				   =					rust_tokenizer.encode(__lowerCamelCase				)
							self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[int]				   =					BasicTokenizer()

							self.assertListEqual(tokenizer.tokenize('''ah\u535A\u63A8zz'''				)       ,  ['''ah''', '''\u535A''', '''\u63A8''', '''zz''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[Any]				   =					BasicTokenizer(do_lower_case=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,  ['''hello''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''hello''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					BasicTokenizer(do_lower_case=__lowerCamelCase       ,  strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''hällo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''h\u00E9llo''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[int]				   =					BasicTokenizer(do_lower_case=__lowerCamelCase       ,  strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''hello''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Dict				   =					BasicTokenizer(do_lower_case=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''hallo''', '''!''', '''how''', '''are''', '''you''', '''?''']				)
							self.assertListEqual(tokenizer.tokenize('''H\u00E9llo'''				)       ,  ['''hello''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					BasicTokenizer(do_lower_case=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU?  '''				)       ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[int]				   =					BasicTokenizer(do_lower_case=__lowerCamelCase       ,  strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''HäLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					BasicTokenizer(do_lower_case=__lowerCamelCase       ,  strip_accents=__lowerCamelCase				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHäLLo!how  \n Are yoU?  '''				)       ,  ['''HaLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Dict				   =					BasicTokenizer(do_lower_case=__lowerCamelCase       ,  never_split=['''[UNK]''']				)

							self.assertListEqual(
							    tokenizer.tokenize(''' \tHeLLo!how  \n Are yoU? [UNK]'''				)       ,  ['''HeLLo''', '''!''', '''how''', '''Are''', '''yoU''', '''?''', '''[UNK]''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :List[Any]				   =					BasicTokenizer()
							UpperCamelCase__  :str				   =					'''a\n\'ll !!to?\'d of, can\'t.'''
							UpperCamelCase__  :List[str]				   =					['''a''', '''\'''', '''ll''', '''!''', '''!''', '''to''', '''?''', '''\'''', '''d''', '''of''', ''',''', '''can''', '''\'''', '''t''', '''.''']
							self.assertListEqual(tokenizer.tokenize(__lowerCamelCase				)       ,  __lowerCamelCase				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :str				   =					['''[UNK]''', '''[CLS]''', '''[SEP]''', '''want''', '''##want''', '''##ed''', '''wa''', '''un''', '''runn''', '''##ing''']

							UpperCamelCase__  :Optional[int]				   =					{}
							for i, token in enumerate(__lowerCamelCase				):
												UpperCamelCase__  :int				   =					i
							UpperCamelCase__  :Tuple				   =					WordpieceTokenizer(vocab=__lowerCamelCase       ,  unk_token='''[UNK]'''				)

							self.assertListEqual(tokenizer.tokenize(''''''				)       ,  []				)

							self.assertListEqual(tokenizer.tokenize('''unwanted running'''				)       ,  ['''un''', '''##want''', '''##ed''', '''runn''', '''##ing''']				)

							self.assertListEqual(tokenizer.tokenize('''unwantedX running'''				)       ,  ['''[UNK]''', '''runn''', '''##ing''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.assertTrue(_is_whitespace(''' '''				)				)
							self.assertTrue(_is_whitespace('''\t'''				)				)
							self.assertTrue(_is_whitespace('''\r'''				)				)
							self.assertTrue(_is_whitespace('''\n'''				)				)
							self.assertTrue(_is_whitespace('''\u00A0'''				)				)

							self.assertFalse(_is_whitespace('''A'''				)				)
							self.assertFalse(_is_whitespace('''-'''				)				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.assertTrue(_is_control('''\u0005'''				)				)

							self.assertFalse(_is_control('''A'''				)				)
							self.assertFalse(_is_control(''' '''				)				)
							self.assertFalse(_is_control('''\t'''				)				)
							self.assertFalse(_is_control('''\r'''				)				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.assertTrue(_is_punctuation('''-'''				)				)
							self.assertTrue(_is_punctuation('''$'''				)				)
							self.assertTrue(_is_punctuation('''`'''				)				)
							self.assertTrue(_is_punctuation('''.'''				)				)

							self.assertFalse(_is_punctuation('''A'''				)				)
							self.assertFalse(_is_punctuation(''' '''				)				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :str				   =					self.get_tokenizer()
							UpperCamelCase__  :Optional[int]				   =					self.get_rust_tokenizer()

							# Example taken from the issue https://github.com/huggingface/tokenizers/issues/340
							self.assertListEqual([tokenizer.tokenize(__lowerCamelCase				) for t in ['''Test''', '''\xad''', '''test''']]       ,  [['''[UNK]'''], [], ['''[UNK]''']]				)

							self.assertListEqual(
							    [rust_tokenizer.tokenize(__lowerCamelCase				) for t in ['''Test''', '''\xad''', '''test''']]       ,  [['''[UNK]'''], [], ['''[UNK]''']]				)




		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :str				   =					self.tokenizer_class.from_pretrained('''bert-base-uncased'''				)

							UpperCamelCase__  :List[Any]				   =					tokenizer.encode('''sequence builders'''       ,  add_special_tokens=__lowerCamelCase				)
							UpperCamelCase__  :List[str]				   =					tokenizer.encode('''multi-sequence build'''       ,  add_special_tokens=__lowerCamelCase				)

							UpperCamelCase__  :List[str]				   =					tokenizer.build_inputs_with_special_tokens(__lowerCamelCase				)
							UpperCamelCase__  :Optional[int]				   =					tokenizer.build_inputs_with_special_tokens(__lowerCamelCase       ,  __lowerCamelCase				)

							assert encoded_sentence == [101] + text + [102]
							assert encoded_pair == [101] + text + [102] + text_a + [102]




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
																	UpperCamelCase__  :List[str]				   =					self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,  **__lowerCamelCase				)

																	UpperCamelCase__  :str				   =					F'''A, naïve {tokenizer_r.mask_token} AllenNLP sentence.'''
																	UpperCamelCase__  :Union[str, Any]				   =					tokenizer_r.encode_plus(
																	    __lowerCamelCase       ,  return_attention_mask=__lowerCamelCase       ,  return_token_type_ids=__lowerCamelCase       ,  return_offsets_mapping=__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase       ,  )

																	UpperCamelCase__  :Dict				   =					tokenizer_r.do_lower_case if hasattr(__lowerCamelCase       ,  '''do_lower_case'''				) else False
																	UpperCamelCase__  :Dict				   =					(
																	    [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''A'''),
																	        ((1, 2), ''','''),
																	        ((3, 5), '''na'''),
																	        ((5, 6), '''##ï'''),
																	        ((6, 8), '''##ve'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''Allen'''),
																	        ((21, 23), '''##NL'''),
																	        ((23, 24), '''##P'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	    if not do_lower_case
																	    else [
																	        ((0, 0), tokenizer_r.cls_token),
																	        ((0, 1), '''a'''),
																	        ((1, 2), ''','''),
																	        ((3, 8), '''naive'''),
																	        ((9, 15), tokenizer_r.mask_token),
																	        ((16, 21), '''allen'''),
																	        ((21, 23), '''##nl'''),
																	        ((23, 24), '''##p'''),
																	        ((25, 33), '''sentence'''),
																	        ((33, 34), '''.'''),
																	        ((0, 0), tokenizer_r.sep_token),
																	    ]
																	)

																	self.assertEqual(
																	    [e[1] for e in expected_results]       ,  tokenizer_r.convert_ids_to_tokens(tokens['''input_ids''']				)				)
																	self.assertEqual([e[0] for e in expected_results]       ,  tokens['''offset_mapping''']				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					['''的''', '''人''', '''有''']
							UpperCamelCase__  :List[Any]				   =					''''''.join(__lowerCamelCase				)
							for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
												with self.subTest(F'''{tokenizer.__class__.__name__} ({pretrained_name})'''				):
																	UpperCamelCase__  :Union[str, Any]				   =					True
																	UpperCamelCase__  :int				   =					self.tokenizer_class.from_pretrained(__lowerCamelCase       ,  **__lowerCamelCase				)
																	UpperCamelCase__  :Optional[Any]				   =					self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,  **__lowerCamelCase				)

																	UpperCamelCase__  :Optional[Any]				   =					tokenizer_p.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)
																	UpperCamelCase__  :str				   =					tokenizer_r.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)

																	UpperCamelCase__  :List[Any]				   =					tokenizer_r.convert_ids_to_tokens(__lowerCamelCase				)
																	UpperCamelCase__  :List[Any]				   =					tokenizer_p.convert_ids_to_tokens(__lowerCamelCase				)

																	# it is expected that each Chinese character is not preceded by "##"
																	self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)
																	self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)

																	UpperCamelCase__  :List[Any]				   =					False
																	UpperCamelCase__  :Union[str, Any]				   =					self.rust_tokenizer_class.from_pretrained(__lowerCamelCase       ,  **__lowerCamelCase				)
																	UpperCamelCase__  :Tuple				   =					self.tokenizer_class.from_pretrained(__lowerCamelCase       ,  **__lowerCamelCase				)

																	UpperCamelCase__  :List[str]				   =					tokenizer_r.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)
																	UpperCamelCase__  :str				   =					tokenizer_p.encode(__lowerCamelCase       ,  add_special_tokens=__lowerCamelCase				)

																	UpperCamelCase__  :Optional[int]				   =					tokenizer_r.convert_ids_to_tokens(__lowerCamelCase				)
																	UpperCamelCase__  :Dict				   =					tokenizer_p.convert_ids_to_tokens(__lowerCamelCase				)

																	# it is expected that only the first Chinese character is not preceded by "##".
																	UpperCamelCase__  :Any				   =					[
																	    F'''##{token}''' if idx != 0 else token for idx, token in enumerate(__lowerCamelCase				)
																	]
																	self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)
																	self.assertListEqual(__lowerCamelCase       ,  __lowerCamelCase				)






"""simple docstring"""



import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


a_					=     get_tests_dir("""fixtures/spiece.model""")






@require_sentencepiece
@require_tokenizers
class 						__snake_case   (						SCREAMING_SNAKE_CASE__				,				unittest.TestCase	):





 """simple docstring"""


 _lowerCamelCase								=	AlbertTokenizer
 _lowerCamelCase								=	AlbertTokenizerFast
 _lowerCamelCase								=	True
 _lowerCamelCase								=	True
 _lowerCamelCase								=	True

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     super().setUp()

     # We have a SentencePiece fixture for testing
     __A       :		str							=	AlbertTokenizer(__lowerCamelCase    )
     tokenizer.save_pretrained(self.tmpdirname    )

 def    UpperCamelCase__(       self   ,		__lowerCamelCase    ):



     '''simple docstring'''




     __A       :		Tuple							=	'''this is a test'''
     __A       :		Union[str, Any]							=	'''this is a test'''
     return input_text, output_text

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		int							=	'''<pad>'''
     __A       :		Optional[int]							=	0

     self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase    )   ,		__lowerCamelCase    )
     self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase    )   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		str							=	list(self.get_tokenizer().get_vocab().keys()    )

     self.assertEqual(vocab_keys[0]   ,		'''<pad>'''    )
     self.assertEqual(vocab_keys[1]   ,		'''<unk>'''    )
     self.assertEqual(vocab_keys[-1]   ,		'''▁eloquent'''    )
     self.assertEqual(len(__lowerCamelCase    )   ,		3_0000    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     self.assertEqual(self.get_tokenizer().vocab_size   ,		3_0000    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     if not self.test_rust_tokenizer:
         return

     __A       :		Dict							=	self.get_tokenizer()
     __A       :		List[Any]							=	self.get_rust_tokenizer()

     __A       :		int							=	'''I was born in 92000, and this is falsé.'''

     __A       :		Any							=	tokenizer.tokenize(__lowerCamelCase    )
     __A       :		Dict							=	rust_tokenizer.tokenize(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

     __A       :		List[Any]							=	tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
     __A       :		List[str]							=	rust_tokenizer.encode(__lowerCamelCase   ,		add_special_tokens=__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

     __A       :		List[Any]							=	self.get_rust_tokenizer()
     __A       :		Union[str, Any]							=	tokenizer.encode(__lowerCamelCase    )
     __A       :		List[Any]							=	rust_tokenizer.encode(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		__lowerCamelCase    )

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		List[str]							=	AlbertTokenizer(__lowerCamelCase   ,		keep_accents=__lowerCamelCase    )

     __A       :		Optional[int]							=	tokenizer.tokenize('''This is a test'''    )
     self.assertListEqual(__lowerCamelCase   ,		['''▁this''', '''▁is''', '''▁a''', '''▁test''']    )

     self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase    )   ,		[48, 25, 21, 1289]    )

     __A       :		Tuple							=	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''    )
     self.assertListEqual(
         __lowerCamelCase   ,		['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']    )
     __A       :		Optional[Any]							=	tokenizer.convert_tokens_to_ids(__lowerCamelCase    )
     self.assertListEqual(__lowerCamelCase   ,		[31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]    )

     __A       :		Optional[Any]							=	tokenizer.convert_ids_to_tokens(__lowerCamelCase    )
     self.assertListEqual(
         __lowerCamelCase   ,		['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']   ,		)

 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Optional[Any]							=	AlbertTokenizer(__lowerCamelCase    )

     __A       :		List[Any]							=	tokenizer.encode('''sequence builders'''    )
     __A       :		str							=	tokenizer.encode('''multi-sequence build'''    )

     __A       :		Union[str, Any]							=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    )
     __A       :		Optional[int]							=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase   ,		__lowerCamelCase    )

     assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
     assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
         tokenizer.sep_token_id
     ]







 @slow
 def    UpperCamelCase__(       self    ):



     '''simple docstring'''




     __A       :		Dict							=	{'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
     # fmt: on

     self.tokenizer_integration_test_util(
         expected_encoding=__lowerCamelCase   ,		model_name='''albert-base-v2'''   ,		revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''   ,		)




