



import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


lowerCamelCase		       =      get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
						import sentencepiece as sp


lowerCamelCase		       =      5
lowerCamelCase		       =      10


@require_sentencepiece
@require_tokenizers
class 				_a						(       SCREAMING_SNAKE_CASE       ,  unittest.TestCase			):





							'''simple docstring'''

							A				:str =       SpeechaTextTokenizer
							A				:Union[str, Any] =       False
							A				:Optional[int] =       True



							def 		_A	(							self				):







											"""simple docstring"""



											super().setUp()

											a__	:							Union[str, Any]      						= sp.SentencePieceProcessor()
											spm_model.Load(__UpperCAmelCase				)
											a__	:							List[Any]      						= ["<s>", "<pad>", "</s>", "<unk>"]

											vocab += [spm_model.IdToPiece(id_				) for id_ in range(len(__UpperCAmelCase				)				)]
											a__	:							Union[str, Any]      						= dict(zip(__UpperCAmelCase ,					range(len(__UpperCAmelCase				)				)				)				)

											a__	:							Tuple      						= Path(self.tmpdirname				)
											save_json(__UpperCAmelCase ,					save_dir / VOCAB_FILES_NAMES["vocab_file"]				)
											if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
															copyfile(__UpperCAmelCase ,					save_dir / VOCAB_FILES_NAMES["spm_file"]				)

											a__	:							Any      						= SpeechaTextTokenizer.from_pretrained(self.tmpdirname				)
											tokenizer.save_pretrained(self.tmpdirname				)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							int      						= "<pad>"
											a__	:							List[Any]      						= 1

											self.assertEqual(self.get_tokenizer()._convert_token_to_id(__UpperCAmelCase				) ,					__UpperCAmelCase				)
											self.assertEqual(self.get_tokenizer()._convert_id_to_token(__UpperCAmelCase				) ,					__UpperCAmelCase				)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Optional[Any]      						= list(self.get_tokenizer().get_vocab().keys()				)

											self.assertEqual(vocab_keys[0] ,					"<s>"				)
											self.assertEqual(vocab_keys[1] ,					"<pad>"				)
											self.assertEqual(vocab_keys[-1] ,					"j"				)
											self.assertEqual(len(__UpperCAmelCase				) ,					1001				)



							def 		_A	(							self				):







											"""simple docstring"""



											self.assertEqual(self.get_tokenizer().vocab_size ,					1001				)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Dict      						= SpeechaTextTokenizer.from_pretrained(self.tmpdirname				)

											a__	:							List[str]      						= tokenizer.tokenize("This is a test"				)
											self.assertListEqual(__UpperCAmelCase ,					["▁This", "▁is", "▁a", "▁t", "est"]				)

											self.assertListEqual(
											    tokenizer.convert_tokens_to_ids(__UpperCAmelCase				) ,					[289, 50, 14, 174, 386] ,					)

											a__	:							List[str]      						= tokenizer.tokenize("I was born in 92000, and this is falsé."				)
											self.assertListEqual(
											    __UpperCAmelCase ,					[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."] ,					)
											a__	:							Optional[int]      						= tokenizer.convert_tokens_to_ids(__UpperCAmelCase				)
											self.assertListEqual(__UpperCAmelCase ,					[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]				)

											a__	:							Union[str, Any]      						= tokenizer.convert_ids_to_tokens(__UpperCAmelCase				)
											self.assertListEqual(
											    __UpperCAmelCase ,					[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."] ,					)




							@slow
							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Optional[int]      						= {"input_ids": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
											# fmt: on

											self.tokenizer_integration_test_util(
											    expected_encoding=__UpperCAmelCase ,					model_name="facebook/s2t-small-mustc-en-de-st" ,					revision="a14f04cf0776c02f62a8cb800cf7909e15ea23ad" ,					)





@require_sentencepiece
class 				_a						(       unittest.TestCase			):





							'''simple docstring'''

							A				:Dict =       "valhalla/s2t_mustc_multilinguial_medium"

							A				:Any =       "C'est trop cool"
							A				:Optional[Any] =       "Esto es genial"



							@classmethod
							def 		_A	(							cls				):







											"""simple docstring"""



											a__	:							SpeechaTextTokenizer      						= SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name				)
											return cls



							def 		_A	(							self				):







											"""simple docstring"""



											self.assertEqual(self.tokenizer.lang_code_to_id["pt"] ,					4				)
											self.assertEqual(self.tokenizer.lang_code_to_id["ru"] ,					6				)
											self.assertEqual(self.tokenizer.lang_code_to_id["it"] ,					9				)
											self.assertEqual(self.tokenizer.lang_code_to_id["de"] ,					11				)



							def 		_A	(							self				):







											"""simple docstring"""



											self.assertEqual(self.tokenizer.vocab_size ,					1_0000				)



							def 		_A	(							self				):







											"""simple docstring"""



											self.assertIn(__UpperCAmelCase ,					self.tokenizer.all_special_ids				)
											a__	:							Optional[int]      						= [ES_CODE, 4, 1601, 47, 7647, 2]
											a__	:							Tuple      						= self.tokenizer.decode(__UpperCAmelCase ,					skip_special_tokens=__UpperCAmelCase				)
											a__	:							List[Any]      						= self.tokenizer.decode(generated_ids[1:] ,					skip_special_tokens=__UpperCAmelCase				)
											self.assertEqual(__UpperCAmelCase ,					__UpperCAmelCase				)
											self.assertNotIn(self.tokenizer.eos_token ,					__UpperCAmelCase				)



							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Tuple      						= "fr"
											a__	:							Any      						= self.tokenizer(self.french_text				).input_ids
											self.assertEqual(encoded[0] ,					__UpperCAmelCase				)
											self.assertEqual(encoded[-1] ,					self.tokenizer.eos_token_id				)




							def 		_A	(							self				):







											"""simple docstring"""



											a__	:							Tuple      						= "fr"
											self.assertListEqual(self.tokenizer.prefix_tokens ,					[FR_CODE]				)

											a__	:							Union[str, Any]      						= "es"
											self.assertListEqual(self.tokenizer.prefix_tokens ,					[ES_CODE]				)







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_sentencepiece_available,
    is_speech_available,
    is_torch_available,
)


lowerCamelCase		       =      {
    """configuration_trocr""": ["""TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP""", """TrOCRConfig"""],
    """processing_trocr""": ["""TrOCRProcessor"""],
}


try:
						if not is_torch_available():
												raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
						pass
else:
						lowerCamelCase		       =      [
						    """TROCR_PRETRAINED_MODEL_ARCHIVE_LIST""",
						    """TrOCRForCausalLM""",
						    """TrOCRPreTrainedModel""",
						]


if TYPE_CHECKING:
						from .configuration_trocr import TROCR_PRETRAINED_CONFIG_ARCHIVE_MAP, TrOCRConfig
						from .processing_trocr import TrOCRProcessor

						try:
												if not is_torch_available():
																		raise OptionalDependencyNotAvailable()
						except OptionalDependencyNotAvailable:
												pass
						else:
												from .modeling_trocr import TROCR_PRETRAINED_MODEL_ARCHIVE_LIST, TrOCRForCausalLM, TrOCRPreTrainedModel

else:
						import sys

						lowerCamelCase		       =      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)


