





'''simple docstring'''






import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


__UpperCamelCase					    =	get_tests_dir("fixtures/test_sentencepiece.model")

if is_sentencepiece_available():
	import sentencepiece as sp


__UpperCamelCase					    =	5
__UpperCamelCase					    =	10




@require_sentencepiece
@require_tokenizers
class  _A (					__lowercase						,					unittest.TestCase		):
		lowercase__:   Optional[Any]				     =   SpeechaTextTokenizer
		lowercase__:   Optional[int]				     =   False
		lowercase__:   Union[str, Any]				     =   True

		def 	lowercase__     (   self					:   List[Any]			)					->    str:


					"""simple docstring"""

					super().setUp()

					__snake_case					:      Tuple							      =     sp.SentencePieceProcessor()
					spm_model.Load(__magic_name__			)
					__snake_case					:      int							      =     ["""<s>""", """<pad>""", """</s>""", """<unk>"""]

					vocab += [spm_model.IdToPiece(id_			) for id_ in range(len(__magic_name__			)			)]
					__snake_case					:      int							      =     dict(zip(__magic_name__  ,      range(len(__magic_name__			)			)			)			)

					__snake_case					:      Union[str, Any]							      =     Path(self.tmpdirname			)
					save_json(__magic_name__  ,      save_dir / VOCAB_FILES_NAMES["""vocab_file"""]			)
					if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
								copyfile(__magic_name__  ,      save_dir / VOCAB_FILES_NAMES["""spm_file"""]			)

					__snake_case					:      Dict							      =     SpeechaTextTokenizer.from_pretrained(self.tmpdirname			)
					tokenizer.save_pretrained(self.tmpdirname			)

		def 	lowercase__     (   self					:   Dict			)					->    int:


					"""simple docstring"""

					__snake_case					:      Any							      =     """<pad>"""
					__snake_case					:      Optional[Any]							      =     1

					self.assertEqual(self.get_tokenizer()._convert_token_to_id(__magic_name__			)  ,      __magic_name__			)
					self.assertEqual(self.get_tokenizer()._convert_id_to_token(__magic_name__			)  ,      __magic_name__			)

		def 	lowercase__     (   self					:   List[str]			)					->    List[str]:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     list(self.get_tokenizer().get_vocab().keys()			)

					self.assertEqual(vocab_keys[0]  ,      """<s>"""			)
					self.assertEqual(vocab_keys[1]  ,      """<pad>"""			)
					self.assertEqual(vocab_keys[-1]  ,      """j"""			)
					self.assertEqual(len(__magic_name__			)  ,      10_01			)

		def 	lowercase__     (   self					:   List[str]			)					->    int:


					"""simple docstring"""

					self.assertEqual(self.get_tokenizer().vocab_size  ,      10_01			)

		def 	lowercase__     (   self					:   str			)					->    Tuple:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     SpeechaTextTokenizer.from_pretrained(self.tmpdirname			)

					__snake_case					:      List[Any]							      =     tokenizer.tokenize("""This is a test"""			)
					self.assertListEqual(__magic_name__  ,      ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]			)

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__magic_name__			)  ,      [2_89, 50, 14, 1_74, 3_86]  ,      )

					__snake_case					:      Optional[int]							      =     tokenizer.tokenize("""I was born in 92000, and this is falsé."""			)
					self.assertListEqual(
					    __magic_name__  ,      [SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """9""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """é""", """."""]  ,      )
					__snake_case					:      Union[str, Any]							      =     tokenizer.convert_tokens_to_ids(__magic_name__			)
					self.assertListEqual(__magic_name__  ,      [12, 25, 88, 59, 28, 23, 11, 4, 6_06, 3_51, 3_51, 3_51, 7, 16, 70, 50, 76, 84, 10, 4, 8]			)

					__snake_case					:      List[Any]							      =     tokenizer.convert_ids_to_tokens(__magic_name__			)
					self.assertListEqual(
					    __magic_name__  ,      [SPIECE_UNDERLINE + """I""", SPIECE_UNDERLINE + """was""", SPIECE_UNDERLINE + """b""", """or""", """n""", SPIECE_UNDERLINE + """in""", SPIECE_UNDERLINE + """""", """<unk>""", """2""", """0""", """0""", """0""", """,""", SPIECE_UNDERLINE + """and""", SPIECE_UNDERLINE + """this""", SPIECE_UNDERLINE + """is""", SPIECE_UNDERLINE + """f""", """al""", """s""", """<unk>""", """."""]  ,      )

		@slow
		def 	lowercase__     (   self					:   int			)					->    Tuple:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     {"""input_ids""": [[37_91, 7_97, 31, 11, 64, 7_97, 31, 24_29, 4_33, 12, 11_76, 12, 20, 7_86, 9_15, 1_42, 24_13, 2_40, 37, 32_38, 7_97, 31, 11, 35, 93, 9_15, 1_42, 24_13, 2_40, 37, 55_40, 5_67, 12_76, 93, 37, 6_10, 40, 62, 4_55, 6_57, 10_42, 1_23, 7_80, 1_77, 37, 3_09, 2_41, 12_98, 5_14, 20, 2_92, 27_37, 1_14, 24_69, 2_41, 85, 64, 3_02, 5_48, 5_28, 4_23, 4, 5_09, 4_06, 4_23, 37, 6_01, 4, 7_77, 3_02, 5_48, 5_28, 4_23, 2_84, 4, 33_88, 5_11, 4_59, 4, 35_55, 40, 3_21, 3_02, 7_05, 4, 33_88, 5_11, 5_83, 3_26, 5, 5, 5, 62, 33_10, 5_60, 1_77, 26_80, 2_17, 15_08, 32, 31, 8_53, 4_18, 64, 5_83, 5_11, 16_05, 62, 35, 93, 5_60, 1_77, 26_80, 2_17, 15_08, 15_21, 64, 5_83, 5_11, 5_19, 62, 20, 15_15, 7_64, 20, 1_49, 2_61, 56_25, 79_72, 20, 55_40, 5_67, 12_76, 93, 39_25, 16_75, 11, 15, 8_02, 79_72, 5_76, 2_17, 15_08, 11, 35, 93, 12_53, 24_41, 15, 2_89, 6_52, 31, 4_16, 3_21, 38_42, 1_15, 40, 9_11, 8, 4_76, 6_19, 4, 3_80, 1_42, 4_23, 3_35, 2_40, 35, 93, 2_64, 8, 11, 3_35, 5_69, 4_20, 1_63, 5, 2], [2_60, 5_48, 5_28, 4_23, 20, 4_51, 20, 26_81, 11_53, 34_34, 20, 55_40, 37, 5_67, 1_26, 12_53, 24_41, 33_76, 4_49, 2_10, 4_31, 15_63, 1_77, 7_67, 55_40, 11, 12_03, 4_72, 11, 29_53, 6_85, 2_85, 3_64, 7_06, 11_53, 20, 67_99, 20, 28_69, 20, 44_64, 1_26, 40, 24_29, 20, 10_40, 8_66, 26_64, 4_18, 20, 3_18, 20, 17_26, 1_86, 20, 2_65, 5_22, 35, 93, 21_91, 46_34, 20, 10_40, 12, 67_99, 15, 2_28, 23_56, 1_42, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [25_75, 26_66, 6_84, 15_82, 11_76, 12, 6_27, 1_49, 6_19, 20, 49_02, 5_63, 11, 20, 1_49, 2_61, 34_20, 23_56, 1_74, 1_42, 47_14, 1_31, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
					# fmt: on

					self.tokenizer_integration_test_util(
					    expected_encoding=__magic_name__  ,      model_name="""facebook/s2t-small-mustc-en-de-st"""  ,      revision="""a14f04cf0776c02f62a8cb800cf7909e15ea23ad"""  ,      )





@require_sentencepiece
class  _A (					unittest.TestCase		):
		lowercase__:   List[Any]				     =   '''valhalla/s2t_mustc_multilinguial_medium'''

		lowercase__:   Union[str, Any]				     =   '''C\'est trop cool'''
		lowercase__:   Union[str, Any]				     =   '''Esto es genial'''

		@classmethod
		def 	lowercase__     (   cls					:   Optional[Any]			)					->    int:


					"""simple docstring"""

					__snake_case					:      SpeechaTextTokenizer							      =     SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name			)
					return cls

		def 	lowercase__     (   self					:   List[Any]			)					->    Optional[int]:


					"""simple docstring"""

					self.assertEqual(self.tokenizer.lang_code_to_id["""pt"""]  ,      4			)
					self.assertEqual(self.tokenizer.lang_code_to_id["""ru"""]  ,      6			)
					self.assertEqual(self.tokenizer.lang_code_to_id["""it"""]  ,      9			)
					self.assertEqual(self.tokenizer.lang_code_to_id["""de"""]  ,      11			)

		def 	lowercase__     (   self					:   Optional[int]			)					->    Dict:


					"""simple docstring"""

					self.assertEqual(self.tokenizer.vocab_size  ,      1_00_00			)

		def 	lowercase__     (   self					:   List[Any]			)					->    Optional[Any]:


					"""simple docstring"""

					self.assertIn(__magic_name__  ,      self.tokenizer.all_special_ids			)
					__snake_case					:      Tuple							      =     [ES_CODE, 4, 16_01, 47, 76_47, 2]
					__snake_case					:      Any							      =     self.tokenizer.decode(__magic_name__  ,      skip_special_tokens=__magic_name__			)
					__snake_case					:      int							      =     self.tokenizer.decode(generated_ids[1:]  ,      skip_special_tokens=__magic_name__			)
					self.assertEqual(__magic_name__  ,      __magic_name__			)
					self.assertNotIn(self.tokenizer.eos_token  ,      __magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Any:


					"""simple docstring"""

					__snake_case					:      Optional[Any]							      =     """fr"""
					__snake_case					:      int							      =     self.tokenizer(self.french_text			).input_ids
					self.assertEqual(encoded[0]  ,      __magic_name__			)
					self.assertEqual(encoded[-1]  ,      self.tokenizer.eos_token_id			)

		def 	lowercase__     (   self					:   Dict			)					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      Dict							      =     """fr"""
					self.assertListEqual(self.tokenizer.prefix_tokens  ,      [FR_CODE]			)

					__snake_case					:      Any							      =     """es"""
					self.assertListEqual(self.tokenizer.prefix_tokens  ,      [ES_CODE]			)









'''simple docstring'''






import doctest
import logging
import os
import unittest
from pathlib import Path
from typing import List, Union

import transformers
from transformers.testing_utils import require_tf, require_torch, slow


__UpperCamelCase					    =	logging.getLogger()




@unittest.skip('''Temporarily disable the doc tests.'''		)
@require_torch
@require_tf
@slow
class  _A (					unittest.TestCase		):

		def 	lowercase__     (   self					:   Optional[int]  ,      __magic_name__					:   Path  ,      __magic_name__					:   Union[str, None] = None  ,      __magic_name__					:   Union[List[str], None] = None  ,      __magic_name__					:   Union[str, List[str], None] = None  ,      __magic_name__					:   bool = True  ,      )					->    Optional[int]:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     [file for file in os.listdir(__magic_name__			) if os.path.isfile(os.path.join(__magic_name__  ,      __magic_name__			)			)]

					if identifier is not None:
								__snake_case					:      List[Any]							      =     [file for file in files if identifier in file]

					if n_identifier is not None:
								if isinstance(__magic_name__  ,      __magic_name__			):
											for n_ in n_identifier:
														__snake_case					:      Optional[int]							      =     [file for file in files if n_ not in file]
								else:
											__snake_case					:      Tuple							      =     [file for file in files if n_identifier not in file]

					__snake_case					:      Dict							      =     ignore_files or []
					ignore_files.append("""__init__.py"""			)
					__snake_case					:      List[str]							      =     [file for file in files if file not in ignore_files]

					for file in files:
								# Open all files
								print("""Testing"""  ,      __magic_name__			)

								if only_modules:
											__snake_case					:      List[Any]							      =     file.split("""."""			)[0]
											try:
														__snake_case					:      List[Any]							      =     getattr(__magic_name__  ,      __magic_name__			)
														__snake_case					:      Union[str, Any]							      =     doctest.DocTestSuite(__magic_name__			)
														__snake_case					:      Dict							      =     unittest.TextTestRunner().run(__magic_name__			)
														self.assertIs(len(result.failures			)  ,      0			)
											except AttributeError:
														logger.info(f'''{module_identifier} is not a module.'''			)
								else:
											__snake_case					:      Tuple							      =     doctest.testfile(str("""..""" / directory / file			)  ,      optionflags=doctest.ELLIPSIS			)
											self.assertIs(result.failed  ,      0			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    Any:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     Path("""src/transformers"""			)
					__snake_case					:      List[Any]							      =     """modeling"""
					__snake_case					:      Union[str, Any]							      =     [
					    """modeling_ctrl.py""",
					    """modeling_tf_ctrl.py""",
					]
					self.analyze_directory(__magic_name__  ,      identifier=__magic_name__  ,      ignore_files=__magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    int:


					"""simple docstring"""

					__snake_case					:      Union[str, Any]							      =     Path("""src/transformers"""			)
					__snake_case					:      Any							      =     """tokenization"""
					self.analyze_directory(__magic_name__  ,      identifier=__magic_name__			)

		def 	lowercase__     (   self					:   Union[str, Any]			)					->    int:


					"""simple docstring"""

					__snake_case					:      List[Any]							      =     Path("""src/transformers"""			)
					__snake_case					:      List[str]							      =     """configuration"""
					self.analyze_directory(__magic_name__  ,      identifier=__magic_name__			)

		def 	lowercase__     (   self					:   Dict			)					->    Dict:


					"""simple docstring"""

					__snake_case					:      Tuple							      =     Path("""src/transformers"""			)
					__snake_case					:      int							      =     ["""configuration""", """modeling""", """tokenization"""]
					self.analyze_directory(__magic_name__  ,      n_identifier=__magic_name__			)

		def 	lowercase__     (   self					:   Optional[int]			)					->    Union[str, Any]:


					"""simple docstring"""

					__snake_case					:      int							      =     Path("""docs/source"""			)
					__snake_case					:      Optional[int]							      =     ["""favicon.ico"""]
					self.analyze_directory(__magic_name__  ,      ignore_files=__magic_name__  ,      only_modules=__magic_name__			)


