





import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase	: Optional[Any]       =get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
     import sentencepiece as sp


__lowercase	: List[str]       =5
__lowercase	: int       =10

@require_sentencepiece
@require_tokenizers
class   A							(						__lowercase		,   unittest.TestCase    ):
      _snake_case				  =SpeechaTextTokenizer
      _snake_case				  =False
      _snake_case				  =True



      def 						lowerCAmelCase__     (						self:    List[str]							)    ->     Any:
         '''simple docstring'''

         super().setUp()

         UpperCAmelCase_           =sp.SentencePieceProcessor()
         spm_model.Load(_lowerCAmelCase							)
         UpperCAmelCase_           =["<s>", "<pad>", "</s>", "<unk>"]

         vocab += [spm_model.IdToPiece(id_							) for id_ in range(len(_lowerCAmelCase							)							)]
         UpperCAmelCase_           =dict(zip(_lowerCAmelCase		,	range(len(_lowerCAmelCase							)							)							)							)

         UpperCAmelCase_           =Path(self.tmpdirname							)
         save_json(_lowerCAmelCase		,	save_dir / VOCAB_FILES_NAMES["vocab_file"]							)
         if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
            copyfile(_lowerCAmelCase		,	save_dir / VOCAB_FILES_NAMES["spm_file"]							)

         UpperCAmelCase_           =SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)
         tokenizer.save_pretrained(self.tmpdirname							)



      def 						lowerCAmelCase__     (						self:    Any							)    ->     Dict:
         '''simple docstring'''

         UpperCAmelCase_           ="<pad>"
         UpperCAmelCase_           =1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(_lowerCAmelCase							)		,	_lowerCAmelCase							)
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(_lowerCAmelCase							)		,	_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    List[Any]							)    ->     int:
         '''simple docstring'''

         UpperCAmelCase_           =list(self.get_tokenizer().get_vocab().keys()							)

         self.assertEqual(vocab_keys[0]		,	"<s>"							)
         self.assertEqual(vocab_keys[1]		,	"<pad>"							)
         self.assertEqual(vocab_keys[-1]		,	"j"							)
         self.assertEqual(len(_lowerCAmelCase							)		,	1001							)



      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     Tuple:
         '''simple docstring'''

         self.assertEqual(self.get_tokenizer().vocab_size		,	1001							)



      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     Optional[int]:
         '''simple docstring'''

         UpperCAmelCase_           =SpeechaTextTokenizer.from_pretrained(self.tmpdirname							)

         UpperCAmelCase_           =tokenizer.tokenize("This is a test"							)
         self.assertListEqual(_lowerCAmelCase		,	["▁This", "▁is", "▁a", "▁t", "est"]							)

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(_lowerCAmelCase							)		,	[289, 50, 14, 174, 386]		,	)

         UpperCAmelCase_           =tokenizer.tokenize("I was born in 92000, and this is falsé."							)
         self.assertListEqual(
             _lowerCAmelCase		,	[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "9", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "é", "."]		,	)
         UpperCAmelCase_           =tokenizer.convert_tokens_to_ids(_lowerCAmelCase							)
         self.assertListEqual(_lowerCAmelCase		,	[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8]							)

         UpperCAmelCase_           =tokenizer.convert_ids_to_tokens(_lowerCAmelCase							)
         self.assertListEqual(
             _lowerCAmelCase		,	[SPIECE_UNDERLINE + "I", SPIECE_UNDERLINE + "was", SPIECE_UNDERLINE + "b", "or", "n", SPIECE_UNDERLINE + "in", SPIECE_UNDERLINE + "", "<unk>", "2", "0", "0", "0", ",", SPIECE_UNDERLINE + "and", SPIECE_UNDERLINE + "this", SPIECE_UNDERLINE + "is", SPIECE_UNDERLINE + "f", "al", "s", "<unk>", "."]		,	)






      @slow
      def 						lowerCAmelCase__     (						self:    Optional[Any]							)    ->     Union[str, Any]:
         '''simple docstring'''

         UpperCAmelCase_           ={"input_ids": [[3791, 797, 31, 11, 64, 797, 31, 2429, 433, 12, 1176, 12, 20, 786, 915, 142, 2413, 240, 37, 3238, 797, 31, 11, 35, 93, 915, 142, 2413, 240, 37, 5540, 567, 1276, 93, 37, 610, 40, 62, 455, 657, 1042, 123, 780, 177, 37, 309, 241, 1298, 514, 20, 292, 2737, 114, 2469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3388, 511, 459, 4, 3555, 40, 321, 302, 705, 4, 3388, 511, 583, 326, 5, 5, 5, 62, 3310, 560, 177, 2680, 217, 1508, 32, 31, 853, 418, 64, 583, 511, 1605, 62, 35, 93, 560, 177, 2680, 217, 1508, 1521, 64, 583, 511, 519, 62, 20, 1515, 764, 20, 149, 261, 5625, 7972, 20, 5540, 567, 1276, 93, 3925, 1675, 11, 15, 802, 7972, 576, 217, 1508, 11, 35, 93, 1253, 2441, 15, 289, 652, 31, 416, 321, 3842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2681, 1153, 3434, 20, 5540, 37, 567, 126, 1253, 2441, 3376, 449, 210, 431, 1563, 177, 767, 5540, 11, 1203, 472, 11, 2953, 685, 285, 364, 706, 1153, 20, 6799, 20, 2869, 20, 4464, 126, 40, 2429, 20, 1040, 866, 2664, 418, 20, 318, 20, 1726, 186, 20, 265, 522, 35, 93, 2191, 4634, 20, 1040, 12, 6799, 15, 228, 2356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2575, 2666, 684, 1582, 1176, 12, 627, 149, 619, 20, 4902, 563, 11, 20, 149, 261, 3420, 2356, 174, 142, 4714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=_lowerCAmelCase		,	model_name="facebook/s2t-small-mustc-en-de-st"		,	revision="a14f04cf0776c02f62a8cb800cf7909e15ea23ad"		,	)

@require_sentencepiece
class   A							(						unittest.TestCase    ):
      _snake_case				  ='''valhalla/s2t_mustc_multilinguial_medium'''

      _snake_case				  ='''C\'est trop cool'''
      _snake_case				  ='''Esto es genial'''



      @classmethod
      def 						lowerCAmelCase__     (						cls:    str							)    ->     Optional[int]:
         '''simple docstring'''

         UpperCAmelCase_           =SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name							)
         return cls



      def 						lowerCAmelCase__     (						self:    str							)    ->     List[str]:
         '''simple docstring'''

         self.assertEqual(self.tokenizer.lang_code_to_id["pt"]		,	4							)
         self.assertEqual(self.tokenizer.lang_code_to_id["ru"]		,	6							)
         self.assertEqual(self.tokenizer.lang_code_to_id["it"]		,	9							)
         self.assertEqual(self.tokenizer.lang_code_to_id["de"]		,	11							)



      def 						lowerCAmelCase__     (						self:    Any							)    ->     Optional[Any]:
         '''simple docstring'''

         self.assertEqual(self.tokenizer.vocab_size		,	1_0000							)



      def 						lowerCAmelCase__     (						self:    str							)    ->     int:
         '''simple docstring'''

         self.assertIn(_lowerCAmelCase		,	self.tokenizer.all_special_ids							)
         UpperCAmelCase_           =[ES_CODE, 4, 1601, 47, 7647, 2]
         UpperCAmelCase_           =self.tokenizer.decode(_lowerCAmelCase		,	skip_special_tokens=_lowerCAmelCase							)
         UpperCAmelCase_           =self.tokenizer.decode(generated_ids[1:]		,	skip_special_tokens=_lowerCAmelCase							)
         self.assertEqual(_lowerCAmelCase		,	_lowerCAmelCase							)
         self.assertNotIn(self.tokenizer.eos_token		,	_lowerCAmelCase							)



      def 						lowerCAmelCase__     (						self:    Dict							)    ->     Union[str, Any]:
         '''simple docstring'''

         UpperCAmelCase_           ="fr"
         UpperCAmelCase_           =self.tokenizer(self.french_text							).input_ids
         self.assertEqual(encoded[0]		,	_lowerCAmelCase							)
         self.assertEqual(encoded[-1]		,	self.tokenizer.eos_token_id							)






      def 						lowerCAmelCase__     (						self:    List[str]							)    ->     Optional[Any]:
         '''simple docstring'''

         UpperCAmelCase_           ="fr"
         self.assertListEqual(self.tokenizer.prefix_tokens		,	[FR_CODE]							)

         UpperCAmelCase_           ="es"
         self.assertListEqual(self.tokenizer.prefix_tokens		,	[ES_CODE]							)











from string import ascii_uppercase

UpperCAmelCase_							=				{char: i for i, char in enumerate(ascii_uppercase)}
UpperCAmelCase_							=				dict(enumerate(ascii_uppercase))





def  __magic_name__  (				lowercase				,       lowercase				)				-> str:

							"""simple docstring"""



							lowercase_	:   Dict  				=     len(lowercase				)
							lowercase_	:   Tuple  				=     0
							while True:
														if x == i:
																					lowercase_	:   Dict  				=     0
														if len(lowercase				) == len(lowercase				):
																					break
														key += key[i]
														i += 1
							return key





def  __magic_name__  (				lowercase				,       lowercase				)				-> str:

							"""simple docstring"""



							lowercase_	:   Any  				=     """"""
							lowercase_	:   Optional[Any]  				=     0
							for letter in message:
														if letter == " ":
																					cipher_text += " "
														else:
																					lowercase_	:   Any  				=     (dicta[letter] - dicta[key_new[i]]) % 26
																					i += 1
																					cipher_text += dicta[x]
							return cipher_text





def  __magic_name__  (				lowercase				,       lowercase				)				-> str:

							"""simple docstring"""



							lowercase_	:   Optional[Any]  				=     """"""
							lowercase_	:   Optional[Any]  				=     0
							for letter in cipher_text:
														if letter == " ":
																					or_txt += " "
														else:
																					lowercase_	:   Optional[int]  				=     (dicta[letter] + dicta[key_new[i]] + 26) % 26
																					i += 1
																					or_txt += dicta[x]
							return or_txt





def  __magic_name__  (				)				-> None:

							"""simple docstring"""



							lowercase_	:   Union[str, Any]  				=     """THE GERMAN ATTACK"""
							lowercase_	:   str  				=     """SECRET"""
							lowercase_	:   int  				=     generate_key(lowercase				,       lowercase				)
							lowercase_	:   List[str]  				=     cipher_text(lowercase				,       lowercase				)
							print(f"""Encrypted Text = {s}"""				)
							print(f"""Original Text = {original_text(lowercase				,       lowercase				)}"""				)


if __name__ == "__main__":
							import doctest

							doctest.testmod()
							main()