


import unittest

from transformers import AlbertTokenizer, AlbertTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_						:			List[str]													=							get_tests_dir('''fixtures/spiece.model''')

@require_sentencepiece
@require_tokenizers
class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						AlbertTokenizer
						UpperCAmelCase             =						AlbertTokenizerFast
						UpperCAmelCase             =						True
						UpperCAmelCase             =						True
						UpperCAmelCase             =						True




						def 							__UpperCamelCase				(       self					:      str  )							->     Dict:


										"""simple docstring"""





										super().setUp()

										# We have a SentencePiece fixture for testing
										_SCREAMING_SNAKE_CASE			  =AlbertTokenizer(_a  )
										tokenizer.save_pretrained(self.tmpdirname  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      Any  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''this is a test'''
										_SCREAMING_SNAKE_CASE			  ='''this is a test'''
										return input_text, output_text




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''<pad>'''
										_SCREAMING_SNAKE_CASE			  =0

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a  )				,    _a  )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a  )				,    _a  )




						def 							__UpperCamelCase				(       self					:      Any  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =list(self.get_tokenizer().get_vocab().keys()  )

										self.assertEqual(vocab_keys[0]				,    '''<pad>'''  )
										self.assertEqual(vocab_keys[1]				,    '''<unk>'''  )
										self.assertEqual(vocab_keys[-1]				,    '''▁eloquent'''  )
										self.assertEqual(len(_a  )				,    3_0000  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Optional[int]:


										"""simple docstring"""





										self.assertEqual(self.get_tokenizer().vocab_size				,    3_0000  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     List[str]:


										"""simple docstring"""





										if not self.test_rust_tokenizer:
														return

										_SCREAMING_SNAKE_CASE			  =self.get_tokenizer()
										_SCREAMING_SNAKE_CASE			  =self.get_rust_tokenizer()

										_SCREAMING_SNAKE_CASE			  ='''I was born in 92000, and this is falsé.'''

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize(_a  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.tokenize(_a  )
										self.assertListEqual(_a				,    _a  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(_a				,    add_special_tokens=_a  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.encode(_a				,    add_special_tokens=_a  )
										self.assertListEqual(_a				,    _a  )

										_SCREAMING_SNAKE_CASE			  =self.get_rust_tokenizer()
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(_a  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.encode(_a  )
										self.assertListEqual(_a				,    _a  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =AlbertTokenizer(_a				,    keep_accents=_a  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''This is a test'''  )
										self.assertListEqual(_a				,    ['''▁this''', '''▁is''', '''▁a''', '''▁test''']  )

										self.assertListEqual(tokenizer.convert_tokens_to_ids(_a  )				,    [48, 25, 21, 1289]  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
										self.assertListEqual(
										    _a				,    ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''é''', '''.''']  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.convert_tokens_to_ids(_a  )
										self.assertListEqual(_a				,    [31, 23, 386, 19, 561, 3050, 15, 17, 48, 25, 8256, 18, 1, 9]  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.convert_ids_to_tokens(_a  )
										self.assertListEqual(
										    _a				,    ['''▁i''', '''▁was''', '''▁born''', '''▁in''', '''▁9''', '''2000''', ''',''', '''▁and''', '''▁this''', '''▁is''', '''▁fal''', '''s''', '''<unk>''', '''.''']				,    )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =AlbertTokenizer(_a  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''sequence builders'''  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode('''multi-sequence build'''  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a  )
										_SCREAMING_SNAKE_CASE			  =tokenizer.build_inputs_with_special_tokens(_a				,    _a  )

										assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
										assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
										    tokenizer.sep_token_id
										]




						@slow
						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ={'''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''input_ids''': [[2, 2_1970, 13, 5, 6092, 167, 28, 7103, 2153, 673, 8, 7028, 1_2051, 18, 17, 7103, 2153, 673, 8, 3515, 1_8684, 8, 4461, 6, 1927, 297, 8, 1_2060, 2607, 18, 13, 5, 4461, 15, 1_0538, 38, 8, 135, 15, 822, 58, 15, 993, 1_0363, 15, 1460, 8005, 4461, 15, 993, 255, 2328, 9, 9, 9, 6, 26, 1112, 816, 3260, 13, 5, 103, 2377, 6, 17, 1112, 816, 2782, 13, 5, 103, 1_0641, 6, 29, 84, 2512, 2430, 782, 1_8684, 2761, 19, 808, 2430, 2556, 17, 855, 1480, 9477, 4091, 128, 1_1712, 15, 7103, 2153, 673, 17, 2_4883, 9990, 9, 3], [2, 1_1502, 25, 1006, 20, 782, 8, 1_1809, 855, 1732, 1_9393, 1_8667, 37, 367, 2_1018, 69, 1854, 34, 1_1860, 1_9124, 27, 156, 225, 17, 193, 4141, 19, 65, 9124, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [2, 14, 2231, 886, 2385, 1_7659, 84, 14, 1_6792, 1952, 9, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], '''token_type_ids''': [[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=_a				,    model_name='''albert-base-v2'''				,    revision='''6b6560eaf5ff2e250b00c50f380c5389a9c2d82e'''				,    )



from ...utils import logging
from ..ta.modeling_tf_ta import TFTaEncoderModel, TFTaForConditionalGeneration, TFTaModel
from .configuration_mta import MTaConfig


snake_case_						:			List[Any]													=							logging.get_logger(__name__)

snake_case_						:			Optional[Any]													=							'''T5Config'''

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						"mt5"
						UpperCAmelCase             =						MTaConfig

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						"mt5"
						UpperCAmelCase             =						MTaConfig

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						"mt5"
						UpperCAmelCase             =						MTaConfig