


import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


snake_case_						:			Any													=							get_tests_dir('''fixtures/test_sentencepiece.model''')

if is_sentencepiece_available():
						import sentencepiece as sp


snake_case_						:			Dict													=							5
snake_case_						:			List[str]													=							10

@require_sentencepiece
@require_tokenizers
class 						A__							(		UpperCamelCase__   ,     unittest.TestCase  ):
						UpperCAmelCase             =						SpeechaTextTokenizer
						UpperCAmelCase             =						False
						UpperCAmelCase             =						True




						def 							__UpperCamelCase				(       self					:      int  )							->     Union[str, Any]:


										"""simple docstring"""





										super().setUp()

										_SCREAMING_SNAKE_CASE			  =sp.SentencePieceProcessor()
										spm_model.Load(_a  )
										_SCREAMING_SNAKE_CASE			  =['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

										vocab += [spm_model.IdToPiece(id_  ) for id_ in range(len(_a  )  )]
										_SCREAMING_SNAKE_CASE			  =dict(zip(_a				,    range(len(_a  )  )  )  )

										_SCREAMING_SNAKE_CASE			  =Path(self.tmpdirname  )
										save_json(_a				,    save_dir / VOCAB_FILES_NAMES['''vocab_file''']  )
										if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
														copyfile(_a				,    save_dir / VOCAB_FILES_NAMES['''spm_file''']  )

										_SCREAMING_SNAKE_CASE			  =SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )
										tokenizer.save_pretrained(self.tmpdirname  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     Optional[int]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''<pad>'''
										_SCREAMING_SNAKE_CASE			  =1

										self.assertEqual(self.get_tokenizer()._convert_token_to_id(_a  )				,    _a  )
										self.assertEqual(self.get_tokenizer()._convert_id_to_token(_a  )				,    _a  )




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =list(self.get_tokenizer().get_vocab().keys()  )

										self.assertEqual(vocab_keys[0]				,    '''<s>'''  )
										self.assertEqual(vocab_keys[1]				,    '''<pad>'''  )
										self.assertEqual(vocab_keys[-1]				,    '''j'''  )
										self.assertEqual(len(_a  )				,    1001  )




						def 							__UpperCamelCase				(       self					:      str  )							->     List[str]:


										"""simple docstring"""





										self.assertEqual(self.get_tokenizer().vocab_size				,    1001  )




						def 							__UpperCamelCase				(       self					:      str  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SpeechaTextTokenizer.from_pretrained(self.tmpdirname  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''This is a test'''  )
										self.assertListEqual(_a				,    ['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']  )

										self.assertListEqual(
										    tokenizer.convert_tokens_to_ids(_a  )				,    [289, 50, 14, 174, 386]				,    )

										_SCREAMING_SNAKE_CASE			  =tokenizer.tokenize('''I was born in 92000, and this is falsé.'''  )
										self.assertListEqual(
										    _a				,    [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']				,    )
										_SCREAMING_SNAKE_CASE			  =tokenizer.convert_tokens_to_ids(_a  )
										self.assertListEqual(_a				,    [12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]  )

										_SCREAMING_SNAKE_CASE			  =tokenizer.convert_ids_to_tokens(_a  )
										self.assertListEqual(
										    _a				,    [SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']				,    )




						@slow
						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     Optional[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ={'''input_ids''': [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
										# fmt: on

										self.tokenizer_integration_test_util(
										    expected_encoding=_a				,    model_name='''facebook/s2t-small-mustc-en-de-st'''				,    revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''				,    )

@require_sentencepiece
class 						A__							(		unittest.TestCase  ):
						UpperCAmelCase             =						"valhalla/s2t_mustc_multilinguial_medium"

						UpperCAmelCase             =						"C'est trop cool"
						UpperCAmelCase             =						"Esto es genial"




						@classmethod
						def 							__UpperCamelCase				(       cls					:      List[str]  )							->     Union[str, Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name  )
										return cls




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     int:


										"""simple docstring"""





										self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']				,    4  )
										self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']				,    6  )
										self.assertEqual(self.tokenizer.lang_code_to_id['''it''']				,    9  )
										self.assertEqual(self.tokenizer.lang_code_to_id['''de''']				,    11  )




						def 							__UpperCamelCase				(       self					:      Dict  )							->     Any:


										"""simple docstring"""





										self.assertEqual(self.tokenizer.vocab_size				,    1_0000  )




						def 							__UpperCamelCase				(       self					:      Optional[Any]  )							->     Dict:


										"""simple docstring"""





										self.assertIn(_a				,    self.tokenizer.all_special_ids  )
										_SCREAMING_SNAKE_CASE			  =[ES_CODE, 4, 1601, 47, 7647, 2]
										_SCREAMING_SNAKE_CASE			  =self.tokenizer.decode(_a				,    skip_special_tokens=_a  )
										_SCREAMING_SNAKE_CASE			  =self.tokenizer.decode(generated_ids[1:]				,    skip_special_tokens=_a  )
										self.assertEqual(_a				,    _a  )
										self.assertNotIn(self.tokenizer.eos_token				,    _a  )




						def 							__UpperCamelCase				(       self					:      List[Any]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''fr'''
										_SCREAMING_SNAKE_CASE			  =self.tokenizer(self.french_text  ).input_ids
										self.assertEqual(encoded[0]				,    _a  )
										self.assertEqual(encoded[-1]				,    self.tokenizer.eos_token_id  )




						def 							__UpperCamelCase				(       self					:      str  )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ='''fr'''
										self.assertListEqual(self.tokenizer.prefix_tokens				,    [FR_CODE]  )

										_SCREAMING_SNAKE_CASE			  ='''es'''
										self.assertListEqual(self.tokenizer.prefix_tokens				,    [ES_CODE]  )



import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


snake_case_						:			Optional[Any]													=							'''scheduler_config.json'''

class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						1
						UpperCAmelCase             =						2
						UpperCAmelCase             =						3
						UpperCAmelCase             =						4
						UpperCAmelCase             =						5
						UpperCAmelCase             =						6
						UpperCAmelCase             =						7
						UpperCAmelCase             =						8
						UpperCAmelCase             =						9
						UpperCAmelCase             =						10
						UpperCAmelCase             =						11
						UpperCAmelCase             =						12
						UpperCAmelCase             =						13
						UpperCAmelCase             =						14

@dataclass
class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						42

class 						A__							:
						UpperCAmelCase             =						SCHEDULER_CONFIG_NAME
						UpperCAmelCase             =						[]
						UpperCAmelCase             =						True




						@classmethod
						def 							__UpperCamelCase				(       cls					:      List[str]				,    _a					:      Dict[str, Any] = None				,    _a					:      Optional[str] = None				,    _a					:      Optional[Any]=False				,    **_a					:      Dict				,    )							->     Dict:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE  ,				_SCREAMING_SNAKE_CASE			  =cls.load_config(
										    pretrained_model_name_or_path=_a				,    subfolder=_a				,    return_unused_kwargs=_a				,    return_commit_hash=_a				,    **_a				,    )
										return cls.from_config(_a				,    return_unused_kwargs=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      Union[str, os.PathLike]				,    _a					:      bool = False				,    **_a					:      int  )							->     Dict:


										"""simple docstring"""





										self.save_config(save_directory=_a				,    push_to_hub=_a				,    **_a  )




						@property
						def 							__UpperCamelCase				(       self					:      List[str]  )							->     List[Any]:


										"""simple docstring"""





										return self._get_compatibles()




						@classmethod
						def 							__UpperCamelCase				(       cls					:      List[Any]  )							->     str:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =list(set([cls.__name__] + cls._compatibles  )  )
										_SCREAMING_SNAKE_CASE			  =importlib.import_module(__name__.split('''.'''  )[0]  )
										_SCREAMING_SNAKE_CASE			  =[
										    getattr(_a				,    _a  ) for c in compatible_classes_str if hasattr(_a				,    _a  )
										]
										return compatible_classes