

import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__lowercase				      =						get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
      import sentencepiece as sp


__lowercase				      =						5
__lowercase				      =						10

@require_sentencepiece
@require_tokenizers
class    _lowercase				(					__lowerCamelCase,unittest.TestCase			):
   _lowercase     :       Optional[Any]  					=    SpeechaTextTokenizer
   _lowercase     :       List[str]  					=    False
   _lowercase     :       Dict  					=    True

   def     UpperCamelCase					(				self	:    Any       ) ->			Union[str, Any]:
         """simple docstring"""





         super().setUp()

         A_							       =		sp.SentencePieceProcessor()
         spm_model.Load(lowerCamelCase__       )
         A_							       =		['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

         vocab += [spm_model.IdToPiece(id_       ) for id_ in range(len(lowerCamelCase__       )       )]
         A_							       =		dict(zip(lowerCamelCase__			,						range(len(lowerCamelCase__       )       )       )       )

         A_							       =		Path(self.tmpdirname       )
         save_json(lowerCamelCase__			,						save_dir / VOCAB_FILES_NAMES['''vocab_file''']       )
         if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
               copyfile(lowerCamelCase__			,						save_dir / VOCAB_FILES_NAMES['''spm_file''']       )

         A_							       =		SpeechaTextTokenizer.from_pretrained(self.tmpdirname       )
         tokenizer.save_pretrained(self.tmpdirname       )

   def     UpperCamelCase					(				self	:    Dict       ) ->			Optional[int]:
         """simple docstring"""





         A_							       =		'''<pad>'''
         A_							       =		1

         self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowerCamelCase__       )			,						lowerCamelCase__       )
         self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowerCamelCase__       )			,						lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    List[str]       ) ->			Dict:
         """simple docstring"""





         A_							       =		list(self.get_tokenizer().get_vocab().keys()       )

         self.assertEqual(vocab_keys[0]			,						'''<s>'''       )
         self.assertEqual(vocab_keys[1]			,						'''<pad>'''       )
         self.assertEqual(vocab_keys[-1]			,						'''j'''       )
         self.assertEqual(len(lowerCamelCase__       )			,						1_0_0_1       )

   def     UpperCamelCase					(				self	:    int       ) ->			List[str]:
         """simple docstring"""





         self.assertEqual(self.get_tokenizer().vocab_size			,						1_0_0_1       )

   def     UpperCamelCase					(				self	:    Union[str, Any]       ) ->			str:
         """simple docstring"""





         A_							       =		SpeechaTextTokenizer.from_pretrained(self.tmpdirname       )

         A_							       =		tokenizer.tokenize('''This is a test'''       )
         self.assertListEqual(lowerCamelCase__			,						['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']       )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(lowerCamelCase__       )			,						[2_8_9, 5_0, 1_4, 1_7_4, 3_8_6]			,						)

         A_							       =		tokenizer.tokenize('''I was born in 92000, and this is falsé.'''       )
         self.assertListEqual(
             lowerCamelCase__			,						[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']			,						)
         A_							       =		tokenizer.convert_tokens_to_ids(lowerCamelCase__       )
         self.assertListEqual(lowerCamelCase__			,						[1_2, 2_5, 8_8, 5_9, 2_8, 2_3, 1_1, 4, 6_0_6, 3_5_1, 3_5_1, 3_5_1, 7, 1_6, 7_0, 5_0, 7_6, 8_4, 1_0, 4, 8]       )

         A_							       =		tokenizer.convert_ids_to_tokens(lowerCamelCase__       )
         self.assertListEqual(
             lowerCamelCase__			,						[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']			,						)





   @slow
   def     UpperCamelCase					(				self	:    str       ) ->			Optional[Any]:
         """simple docstring"""





         A_							       =		{'''input_ids''': [[3_7_9_1, 7_9_7, 3_1, 1_1, 6_4, 7_9_7, 3_1, 2_4_2_9, 4_3_3, 1_2, 1_1_7_6, 1_2, 2_0, 7_8_6, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 3_2_3_8, 7_9_7, 3_1, 1_1, 3_5, 9_3, 9_1_5, 1_4_2, 2_4_1_3, 2_4_0, 3_7, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_7, 6_1_0, 4_0, 6_2, 4_5_5, 6_5_7, 1_0_4_2, 1_2_3, 7_8_0, 1_7_7, 3_7, 3_0_9, 2_4_1, 1_2_9_8, 5_1_4, 2_0, 2_9_2, 2_7_3_7, 1_1_4, 2_4_6_9, 2_4_1, 8_5, 6_4, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 4, 5_0_9, 4_0_6, 4_2_3, 3_7, 6_0_1, 4, 7_7_7, 3_0_2, 5_4_8, 5_2_8, 4_2_3, 2_8_4, 4, 3_3_8_8, 5_1_1, 4_5_9, 4, 3_5_5_5, 4_0, 3_2_1, 3_0_2, 7_0_5, 4, 3_3_8_8, 5_1_1, 5_8_3, 3_2_6, 5, 5, 5, 6_2, 3_3_1_0, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 3_2, 3_1, 8_5_3, 4_1_8, 6_4, 5_8_3, 5_1_1, 1_6_0_5, 6_2, 3_5, 9_3, 5_6_0, 1_7_7, 2_6_8_0, 2_1_7, 1_5_0_8, 1_5_2_1, 6_4, 5_8_3, 5_1_1, 5_1_9, 6_2, 2_0, 1_5_1_5, 7_6_4, 2_0, 1_4_9, 2_6_1, 5_6_2_5, 7_9_7_2, 2_0, 5_5_4_0, 5_6_7, 1_2_7_6, 9_3, 3_9_2_5, 1_6_7_5, 1_1, 1_5, 8_0_2, 7_9_7_2, 5_7_6, 2_1_7, 1_5_0_8, 1_1, 3_5, 9_3, 1_2_5_3, 2_4_4_1, 1_5, 2_8_9, 6_5_2, 3_1, 4_1_6, 3_2_1, 3_8_4_2, 1_1_5, 4_0, 9_1_1, 8, 4_7_6, 6_1_9, 4, 3_8_0, 1_4_2, 4_2_3, 3_3_5, 2_4_0, 3_5, 9_3, 2_6_4, 8, 1_1, 3_3_5, 5_6_9, 4_2_0, 1_6_3, 5, 2], [2_6_0, 5_4_8, 5_2_8, 4_2_3, 2_0, 4_5_1, 2_0, 2_6_8_1, 1_1_5_3, 3_4_3_4, 2_0, 5_5_4_0, 3_7, 5_6_7, 1_2_6, 1_2_5_3, 2_4_4_1, 3_3_7_6, 4_4_9, 2_1_0, 4_3_1, 1_5_6_3, 1_7_7, 7_6_7, 5_5_4_0, 1_1, 1_2_0_3, 4_7_2, 1_1, 2_9_5_3, 6_8_5, 2_8_5, 3_6_4, 7_0_6, 1_1_5_3, 2_0, 6_7_9_9, 2_0, 2_8_6_9, 2_0, 4_4_6_4, 1_2_6, 4_0, 2_4_2_9, 2_0, 1_0_4_0, 8_6_6, 2_6_6_4, 4_1_8, 2_0, 3_1_8, 2_0, 1_7_2_6, 1_8_6, 2_0, 2_6_5, 5_2_2, 3_5, 9_3, 2_1_9_1, 4_6_3_4, 2_0, 1_0_4_0, 1_2, 6_7_9_9, 1_5, 2_2_8, 2_3_5_6, 1_4_2, 3_1, 1_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_5_7_5, 2_6_6_6, 6_8_4, 1_5_8_2, 1_1_7_6, 1_2, 6_2_7, 1_4_9, 6_1_9, 2_0, 4_9_0_2, 5_6_3, 1_1, 2_0, 1_4_9, 2_6_1, 3_4_2_0, 2_3_5_6, 1_7_4, 1_4_2, 4_7_1_4, 1_3_1, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
         # fmt: on

         self.tokenizer_integration_test_util(
             expected_encoding=lowerCamelCase__			,						model_name='''facebook/s2t-small-mustc-en-de-st'''			,						revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''			,						)



@require_sentencepiece
class    _lowercase				(					unittest.TestCase			):
   _lowercase     :       List[Any]  					=    'valhalla/s2t_mustc_multilinguial_medium'

   _lowercase     :       int  					=    'C\'est trop cool'
   _lowercase     :       Any  					=    'Esto es genial'

   @classmethod
   def     UpperCamelCase					(				cls	:    Optional[Any]       ) ->			int:
         """simple docstring"""





         A_							       =		SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name       )
         return cls

   def     UpperCamelCase					(				self	:    Optional[int]       ) ->			List[Any]:
         """simple docstring"""





         self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']			,						4       )
         self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']			,						6       )
         self.assertEqual(self.tokenizer.lang_code_to_id['''it''']			,						9       )
         self.assertEqual(self.tokenizer.lang_code_to_id['''de''']			,						1_1       )

   def     UpperCamelCase					(				self	:    Optional[Any]       ) ->			Union[str, Any]:
         """simple docstring"""





         self.assertEqual(self.tokenizer.vocab_size			,						1_0_0_0_0       )

   def     UpperCamelCase					(				self	:    List[str]       ) ->			Any:
         """simple docstring"""





         self.assertIn(lowerCamelCase__			,						self.tokenizer.all_special_ids       )
         A_							       =		[ES_CODE, 4, 1_6_0_1, 4_7, 7_6_4_7, 2]
         A_							       =		self.tokenizer.decode(lowerCamelCase__			,						skip_special_tokens=lowerCamelCase__       )
         A_							       =		self.tokenizer.decode(generated_ids[1:]			,						skip_special_tokens=lowerCamelCase__       )
         self.assertEqual(lowerCamelCase__			,						lowerCamelCase__       )
         self.assertNotIn(self.tokenizer.eos_token			,						lowerCamelCase__       )

   def     UpperCamelCase					(				self	:    Optional[int]       ) ->			Union[str, Any]:
         """simple docstring"""





         A_							       =		'''fr'''
         A_							       =		self.tokenizer(self.french_text       ).input_ids
         self.assertEqual(encoded[0]			,						lowerCamelCase__       )
         self.assertEqual(encoded[-1]			,						self.tokenizer.eos_token_id       )





   def     UpperCamelCase					(				self	:    Dict       ) ->			Dict:
         """simple docstring"""





         A_							       =		'''fr'''
         self.assertListEqual(self.tokenizer.prefix_tokens			,						[FR_CODE]       )

         A_							       =		'''es'''
         self.assertListEqual(self.tokenizer.prefix_tokens			,						[ES_CODE]       )








def 							_lowerCamelCase   (			SCREAMING_SNAKE_CASE = 1000   ):
      '''simple docstring'''







      return sum(2 * a * ((a - 1) // 2) for a in range(3		,				n + 1   )   )


if __name__ == "__main__":
      print(solution())





