


import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_						:			Any													=							get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_sentencepiece_available():
							import sentencepiece as sp


snake_case_						:			Dict													=							5
snake_case_						:			List[str]													=							10

@require_sentencepiece
@require_tokenizers
class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						SpeechaTextTokenizer
						UpperCAmelCase             =						False
						UpperCAmelCase             =						True




						def 							__UpperCamelCase				(       self					:      int  )							->     Union[str, Any]:


										"""simple docstring"""





										super().setUp()

										_SCREAMING_SNAKE_CASE			  =sp.SentencePieceProcessor()
										spm_model.Load(_a  )
										_SCREAMING_SNAKE_CASE			  =['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

										vocab += [spm_model.IdToPiece(id_  ) for id_ in range(len(_a  )  )]
										_SCREAMING_SNAKE_CASE			  =dict(zip(_a				,    range(len(_a  )  )  )  )

										_SCREAMING_SNAKE_CASE			  =Path(self.tmpdirname  )
										save_json(_a				,    save_dir / VOCAB_FILES_NAMES['''vocab_file''']  )
										if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
														copyfile(_a				,    save_dir / VOCAB_FILES_NAMES['''spm_file''']  )

										_SCREAMING_SNAKE_CASE			  =SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )
										tokenizer.save_pretrained(self.tmpdirname  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''<pad>'''
										_SCREAMING_SNAKE_CASE			  =1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a  )				,    _a  )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a  )				,    _a  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =list(self.get_tokenizer().get_vocab().keys()  )

										self.assertEqual(vocab_keys[0]				,    '''<s>'''  )
										self.assertEqual(vocab_keys[1]				,    '''<pad>'''  )
										self.assertEqual(vocab_keys[-1]				,    '''j'''  )
										self.assertEqual(len(_a  )				,    1001  )




						def 							__UpperCamelCase				(       self					:      str  )							->     List[str]:


										"""simple docstring"""





										self.assertEqual(self.get_tokenizer().vocab_size				,    1001  )




						def 							__UpperCamelCase				(       self					:      str  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''This is a test'''  )
										self.assertListEqual(_a				,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']  )

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(_a  )				,    [289, 50, 14, 174, 386]				,    )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
										self.assertListEqual(
										    _a				,    [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']				,    )
										_SCREAMING_SNAKE_CASE			  =tokenizer.convert_tokens_to_ids(_a  )
										self.assertListEqual(_a				,    [12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.convert_ids_to_tokens(_a  )
										self.assertListEqual(
										    _a				,    [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']				,    )




						@slow
						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ={'''input_ids''': [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=_a				,    model_name='''facebook/s2t-small-mustc-en-de-st'''				,    revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''				,    )

@require_sentencepiece
class 						A__							(		unittest.TestCase  ):
						UpperCAmelCase             =						"valhalla/s2t_mustc_multilinguial_medium"

						UpperCAmelCase             =						"C'est trop cool"
						UpperCAmelCase             =						"Esto es genial"




						@classmethod
						def 							__UpperCamelCase				(       cls					:      List[str]  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name  )
										return cls




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     int:


										"""simple docstring"""





										self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']				,    4  )
										self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']				,    6  )
										self.assertEqual(self.tokenizer.lang_code_to_id['''it''']				,    9  )
										self.assertEqual(self.tokenizer.lang_code_to_id['''de''']				,    11  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Any:


										"""simple docstring"""





										self.assertEqual(self.tokenizer.vocab_size				,    1_0000  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     Dict:


										"""simple docstring"""





										self.assertIn(_a				,    self.tokenizer.all_special_ids  )
										_SCREAMING_SNAKE_CASE			  =[ES_CODE, 4, 1601, 47, 7647, 2]
										_SCREAMING_SNAKE_CASE			  =self.tokenizer.decode(_a				,    skip_special_tokens=_a  )
										_SCREAMING_SNAKE_CASE			  =self.tokenizer.decode(generated_ids[1:]				,    skip_special_tokens=_a  )
										self.assertEqual(_a				,    _a  )
										self.assertNotIn(self.tokenizer.eos_token				,    _a  )




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''fr'''
										_SCREAMING_SNAKE_CASE			  =self.tokenizer(self.french_text  ).input_ids
										self.assertEqual(encoded[0]				,    _a  )
										self.assertEqual(encoded[-1]				,    self.tokenizer.eos_token_id  )




						def 							__UpperCamelCase				(       self					:      str  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''fr'''
										self.assertListEqual(self.tokenizer.prefix_tokens				,    [FR_CODE]  )

										_SCREAMING_SNAKE_CASE			  ='''es'''
										self.assertListEqual(self.tokenizer.prefix_tokens				,    [ES_CODE]  )



import os
import unittest

from transformers import LxmertTokenizer, LxmertTokenizerFast
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
from transformers.testing_utils import require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin

@require_tokenizers
class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						LxmertTokenizer
						UpperCAmelCase             =						LxmertTokenizerFast
						UpperCAmelCase             =						True
						UpperCAmelCase             =						True




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Tuple:


										"""simple docstring"""





										super().setUp()

										_SCREAMING_SNAKE_CASE			  =[
										    '''[UNK]''',
										    '''[CLS]''',
										    '''[SEP]''',
										    '''want''',
										    '''##want''',
										    '''##ed''',
										    '''wa''',
										    '''un''',
										    '''runn''',
										    '''##ing''',
										    ''',''',
										    '''low''',
										    '''lowest''',
										]
										_SCREAMING_SNAKE_CASE			  =os.path.join(self.tmpdirname				,    VOCAB_FILES_NAMES['''vocab_file''']  )
										with open(self.vocab_file				,    '''w'''				,    encoding='''utf-8'''  ) as vocab_writer:
														vocab_writer.write(''''''.join([x + '''\n''' for x in vocab_tokens]  )  )




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      Tuple  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''UNwant\u00E9d,running'''
										_SCREAMING_SNAKE_CASE			  ='''unwanted, running'''
										return input_text, output_text




						def 							__UpperCamelCase				(       self					:      Tuple  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.tokenizer_class(self.vocab_file  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''UNwant\u00E9d,running'''  )
										self.assertListEqual(_a				,    ['''un''', '''##want''', '''##ed''', ''',''', '''runn''', '''##ing''']  )
										self.assertListEqual(tokenizer.convert_tokens_to_ids(_a  )				,    [7, 4, 5, 10, 8, 9]  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     List[Any]:


										"""simple docstring"""





										if not self.test_rust_tokenizer:
														return

										_SCREAMING_SNAKE_CASE			  =self.get_tokenizer()
										_SCREAMING_SNAKE_CASE			  =self.get_rust_tokenizer()

										_SCREAMING_SNAKE_CASE			  ='''I was born in 92000, and this is falsé.'''

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize(_a  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.tokenize(_a  )
										self.assertListEqual(_a				,    _a  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(_a				,    add_special_tokens=_a  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.encode(_a				,    add_special_tokens=_a  )
										self.assertListEqual(_a				,    _a  )

										_SCREAMING_SNAKE_CASE			  =self.get_rust_tokenizer()
										_SCREAMING_SNAKE_CASE			  =tokenizer.encode(_a  )
										_SCREAMING_SNAKE_CASE			  =rust_tokenizer.encode(_a  )
										self.assertListEqual(_a				,    _a  )