
'''simple docstring'''







import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowercase			:  int   							=	get_tests_dir("fixtures/spiece.model")






@require_sentencepiece
@require_tokenizers
class 			lowerCAmelCase__       (  UpperCamelCase__   ,						unittest.TestCase					):
					lowerCAmelCase_					       =					AlbertTokenizer
					lowerCAmelCase_					       =					AlbertTokenizerFast
					lowerCAmelCase_					       =					True
					lowerCAmelCase_					       =					True
					lowerCAmelCase_					       =					True






					def      _snake_case   (       self		):


											"""simple docstring"""





											super().setUp()

											# We have a SentencePiece fixture for testing
											lowercase_     :					Optional[int]													=	AlbertTokenizer(__lowerCamelCase		)
											tokenizer.save_pretrained(self.tmpdirname		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					int													=	'''this is a test'''
											lowercase_     :					List[Any]													=	'''this is a test'''
											return input_text, output_text






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					str													=	'''<pad>'''
											lowercase_     :					Any													=	0

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__lowerCamelCase		)    ,							__lowerCamelCase		)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__lowerCamelCase		)    ,							__lowerCamelCase		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Any													=	list(self.get_tokenizer().get_vocab().keys()		)

											self.assertEqual(vocab_keys[0]    ,							'''<pad>'''		)
											self.assertEqual(vocab_keys[1]    ,							'''<unk>'''		)
											self.assertEqual(vocab_keys[-1]    ,							'''▁eloquent'''		)
											self.assertEqual(len(__lowerCamelCase		)    ,							3_00_00		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											self.assertEqual(self.get_tokenizer().vocab_size    ,							3_00_00		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											if not self.test_rust_tokenizer:
																	return

											lowercase_     :					int													=	self.get_tokenizer()
											lowercase_     :					Optional[Any]													=	self.get_rust_tokenizer()

											lowercase_     :					List[str]													=	'''I was born in 92000, and this is falsé.'''

											lowercase_     :					List[Any]													=	tokenizer.tokenize(__lowerCamelCase		)
											lowercase_     :					List[Any]													=	rust_tokenizer.tokenize(__lowerCamelCase		)
											self.assertListEqual(__lowerCamelCase    ,							__lowerCamelCase		)

											lowercase_     :					Union[str, Any]													=	tokenizer.encode(__lowerCamelCase    ,							add_special_tokens=__lowerCamelCase		)
											lowercase_     :					List[Any]													=	rust_tokenizer.encode(__lowerCamelCase    ,							add_special_tokens=__lowerCamelCase		)
											self.assertListEqual(__lowerCamelCase    ,							__lowerCamelCase		)

											lowercase_     :					Optional[int]													=	self.get_rust_tokenizer()
											lowercase_     :					Optional[Any]													=	tokenizer.encode(__lowerCamelCase		)
											lowercase_     :					List[Any]													=	rust_tokenizer.encode(__lowerCamelCase		)
											self.assertListEqual(__lowerCamelCase    ,							__lowerCamelCase		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	AlbertTokenizer(__lowerCamelCase    ,							keep_accents=__lowerCamelCase		)

											lowercase_     :					Optional[int]													=	tokenizer.tokenize('''This is a test'''		)
											self.assertListEqual(__lowerCamelCase    ,							['''▁this''', '''▁is''', '''▁a''', '''▁test''']		)

											self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase		)    ,							[48, 25, 21, 12_89]		)

											lowercase_     :					List[str]													=	tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
											self.assertListEqual(
											    __lowerCamelCase    ,							['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']		)
											lowercase_     :					Any													=	tokenizer.convert_tokens_to_ids(__lowerCamelCase		)
											self.assertListEqual(__lowerCamelCase    ,							[31, 23, 3_86, 19, 5_61, 30_50, 15, 17, 48, 25, 82_56, 18, 1, 9]		)

											lowercase_     :					Dict													=	tokenizer.convert_ids_to_tokens(__lowerCamelCase		)
											self.assertListEqual(
											    __lowerCamelCase    ,							['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']    ,							)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	AlbertTokenizer(__lowerCamelCase		)

											lowercase_     :					Optional[Any]													=	tokenizer.encode('''sequence builders'''		)
											lowercase_     :					str													=	tokenizer.encode('''multi-sequence build'''		)

											lowercase_     :					Optional[int]													=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase		)
											lowercase_     :					Any													=	tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,							__lowerCamelCase		)

											assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
											assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
											    tokenizer.sep_token_id
											]






					@slow
					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					int													=	{'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_19_70, 13, 5, 60_92, 1_67, 28, 71_03, 21_53, 6_73, 8, 70_28, 1_20_51, 18, 17, 71_03, 21_53, 6_73, 8, 35_15, 1_86_84, 8, 44_61, 6, 19_27, 2_97, 8, 1_20_60, 26_07, 18, 13, 5, 44_61, 15, 1_05_38, 38, 8, 1_35, 15, 8_22, 58, 15, 9_93, 1_03_63, 15, 14_60, 80_05, 44_61, 15, 9_93, 2_55, 23_28, 9, 9, 9, 6, 26, 11_12, 8_16, 32_60, 13, 5, 1_03, 23_77, 6, 17, 11_12, 8_16, 27_82, 13, 5, 1_03, 1_06_41, 6, 29, 84, 25_12, 24_30, 7_82, 1_86_84, 27_61, 19, 8_08, 24_30, 25_56, 17, 8_55, 14_80, 94_77, 40_91, 1_28, 1_17_12, 15, 71_03, 21_53, 6_73, 17, 2_48_83, 99_90, 9, 3], [2, 1_15_02, 25, 10_06, 20, 7_82, 8, 1_18_09, 8_55, 17_32, 1_93_93, 1_86_67, 37, 3_67, 2_10_18, 69, 18_54, 34, 1_18_60, 1_91_24, 27, 1_56, 2_25, 17, 1_93, 41_41, 19, 65, 91_24, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 22_31, 8_86, 23_85, 1_76_59, 84, 14, 1_67_92, 19_52, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__lowerCamelCase    ,							model_name='''albert-base-v2'''    ,							revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''    ,							)



'''simple docstring'''







def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str							):



						"""simple docstring"""





						return [
						    txt[:a] + txt[a].upper() + txt[a + 1 :]
						    for a in range(len(__SCREAMING_SNAKE_CASE							)							)
						    if txt[a].isalpha()
						]


if __name__ == "__main__":
							__import__("doctest").testmod()



