

import argparse
import collections
import os
import re

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
lowercase			:				Tuple   =     """src/transformers"""
lowercase			:				Any   =     """docs/source/en"""
lowercase			:				Dict   =     """."""






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_     ,				lowerCamelCase_		):

							with open(lowercase_     ,				'''r'''     ,				encoding='''utf-8'''     ,				newline='''\n'''		) as f:
														A      :					int      		=					f.readlines()
							# Find the start prompt.
							A      :					int      		=					0
							while not lines[start_index].startswith(lowercase_		):
														start_index += 1
							start_index += 1

							A      :					str      		=					start_index
							while not lines[end_index].startswith(lowercase_		):
														end_index += 1
							end_index -= 1

							while len(lines[start_index]		) <= 1:
														start_index += 1
							while len(lines[end_index]		) <= 1:
														end_index -= 1
							end_index += 1
							return "".join(lines[start_index:end_index]		), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
lowercase			:				Optional[Any]   =     """Model|Encoder|Decoder|ForConditionalGeneration"""
# Regexes that match TF/Flax/PT model names.
lowercase			:				Optional[Any]   =     re.compile(r"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
lowercase			:				Union[str, Any]   =     re.compile(r"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowercase			:				str   =     re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# This is to make sure the transformers module imported is the one in the repo.
lowercase			:				str   =     direct_transformers_import(TRANSFORMERS_PATH)






def        snake_case__		(    lowerCamelCase_		):
							A      :					List[str]      		=					re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''     ,				lowercase_		)
							return [m.group(0		) for m in matches]






def        snake_case__		(    lowerCamelCase_     ,				lowerCamelCase_		):
							A      :					int      		=					2 if text == '''✅''' or text == '''❌''' else len(lowercase_		)
							A      :					List[str]      		=					(width - text_length) // 2
							A      :					Dict      		=					width - text_length - left_indent
							return " " * left_indent + text + " " * right_indent






def        snake_case__		(    ):
							A      :					Optional[Any]      		=					transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
							A      :					Union[str, Any]      		=					{
							    name: config_maping_names[code]
							    for code, name in transformers_module.MODEL_NAMES_MAPPING.items()
							    if code in config_maping_names
							}
							A      :					Union[str, Any]      		=					{name: config.replace('''Config'''     ,				''''''		) for name, config in model_name_to_config.items()}

							# Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
							A      :					List[str]      		=					collections.defaultdict(lowercase_		)
							A      :					str      		=					collections.defaultdict(lowercase_		)
							A      :					Any      		=					collections.defaultdict(lowercase_		)
							A      :					str      		=					collections.defaultdict(lowercase_		)
							A      :					Dict      		=					collections.defaultdict(lowercase_		)

							# Let's lookup through all transformers object (once).
							for attr_name in dir(lowercase_		):
														A      :					List[str]      		=					None
														if attr_name.endswith('''Tokenizer'''		):
																					A      :					Union[str, Any]      		=					slow_tokenizers
																					A      :					List[Any]      		=					attr_name[:-9]
														elif attr_name.endswith('''TokenizerFast'''		):
																					A      :					Dict      		=					fast_tokenizers
																					A      :					List[Any]      		=					attr_name[:-13]
														elif _re_tf_models.match(lowercase_		) is not None:
																					A      :					str      		=					tf_models
																					A      :					Tuple      		=					_re_tf_models.match(lowercase_		).groups()[0]
														elif _re_flax_models.match(lowercase_		) is not None:
																					A      :					Tuple      		=					flax_models
																					A      :					List[str]      		=					_re_flax_models.match(lowercase_		).groups()[0]
														elif _re_pt_models.match(lowercase_		) is not None:
																					A      :					List[Any]      		=					pt_models
																					A      :					str      		=					_re_pt_models.match(lowercase_		).groups()[0]

														if lookup_dict is not None:
																					while len(lowercase_		) > 0:
																												if attr_name in model_name_to_prefix.values():
																																			A      :					List[str]      		=					True
																																			break
																												# Try again after removing the last word in the name
																												A      :					Union[str, Any]      		=					''''''.join(camel_case_split(lowercase_		)[:-1]		)

    # Let's build that table!
							A      :					List[Any]      		=					list(model_name_to_config.keys()		)
							model_names.sort(key=str.lower		)
							A      :					Union[str, Any]      		=					['''Model''', '''Tokenizer slow''', '''Tokenizer fast''', '''PyTorch support''', '''TensorFlow support''', '''Flax Support''']
							# We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
							A      :					List[str]      		=					[len(lowercase_		) + 2 for c in columns]
							A      :					Optional[Any]      		=					max([len(lowercase_		) for name in model_names]		) + 2

							# Build the table per se
							A      :					List[Any]      		=					'''|''' + '''|'''.join([_center_text(lowercase_     ,				lowercase_		) for c, w in zip(lowercase_     ,				lowercase_		)]		) + '''|\n'''
							# Use ":-----:" format to center-aligned table cell texts
							table += "|" + "|".join([''':''' + '''-''' * (w - 2) + ''':''' for w in widths]		) + "|\n"

							A      :					Optional[Any]      		=					{True: '''✅''', False: '''❌'''}
							for name in model_names:
														A      :					Dict      		=					model_name_to_prefix[name]
														A      :					List[Any]      		=					[
														    name,
														    check[slow_tokenizers[prefix]],
														    check[fast_tokenizers[prefix]],
														    check[pt_models[prefix]],
														    check[tf_models[prefix]],
														    check[flax_models[prefix]],
														]
														table += "|" + "|".join([_center_text(lowercase_     ,				lowercase_		) for l, w in zip(lowercase_     ,				lowercase_		)]		) + "|\n"
							return table






def        snake_case__		(    lowerCamelCase_=False		):
							A		,		A		,		A		,		A      :					Union[str, Any]      		=					_find_text_in_file(
							    filename=os.path.join(lowercase_     ,				'''index.md'''		)     ,				start_prompt='''<!--This table is updated automatically from the auto modules'''     ,				end_prompt='''<!-- End table-->'''     ,				)
							A      :					Dict      		=					get_model_table_from_auto_modules()

							if current_table != new_table:
														if overwrite:
																					with open(os.path.join(lowercase_     ,				'''index.md'''		)     ,				'''w'''     ,				encoding='''utf-8'''     ,				newline='''\n'''		) as f:
																												f.writelines(lines[:start_index] + [new_table] + lines[end_index:]		)
														else:
																					raise ValueError(
																					    '''The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this.'''		)


if __name__ == "__main__":
		lowercase			:				Optional[Any]   =     argparse.ArgumentParser()
		parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")
		lowercase			:				Optional[Any]   =     parser.parse_args()

		check_model_table(args.fix_and_overwrite)












import unittest
from pathlib import Path
from shutil import copyfile

from transformers import SPIECE_UNDERLINE, is_sentencepiece_available
from transformers.models.speech_to_text import SpeechaTextTokenizer
from transformers.models.speech_to_text.tokenization_speech_to_text import VOCAB_FILES_NAMES, save_json
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin


_lowerCamelCase      :						Dict							=  get_tests_dir("""fixtures/test_sentencepiece.model""")

if is_sentencepiece_available():
  import sentencepiece as sp


_lowerCamelCase      :						str							=  5
_lowerCamelCase      :						int							=  10


@require_sentencepiece
@require_tokenizers
class 							UpperCamelCase_       (					UpperCAmelCase__						,   unittest.TestCase	):

    '''simple docstring'''






    UpperCAmelCase__				    = SpeechaTextTokenizer
    UpperCAmelCase__				    = False
    UpperCAmelCase__				    = True


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->List[str]:





           '''simple docstring'''

           super().setUp()

           A__ 					=       sp.SentencePieceProcessor()
           spm_model.Load(UpperCAmelCase__)
           A__ 					=       ['''<s>''', '''<pad>''', '''</s>''', '''<unk>''']

           vocab += [spm_model.IdToPiece(id_) for id_ in range(len(UpperCAmelCase__))]
           A__ 					=       dict(zip(UpperCAmelCase__   ,					range(len(UpperCAmelCase__))))

           A__ 					=       Path(self.tmpdirname)
           save_json(UpperCAmelCase__   ,					save_dir / VOCAB_FILES_NAMES['''vocab_file'''])
           if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
                  copyfile(UpperCAmelCase__   ,					save_dir / VOCAB_FILES_NAMES['''spm_file'''])

           A__ 					=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)
           tokenizer.save_pretrained(self.tmpdirname)


    def 			SCREAMING_SNAKE_CASE				(     self :      Any)		->Optional[int]:





           '''simple docstring'''

           A__ 					=       '''<pad>'''
           A__ 					=       1

           self.assertEqual(self.get_tokenizer()._convert_token_to_id(UpperCAmelCase__)   ,					UpperCAmelCase__)
           self.assertEqual(self.get_tokenizer()._convert_id_to_token(UpperCAmelCase__)   ,					UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->Any:





           '''simple docstring'''

           A__ 					=       list(self.get_tokenizer().get_vocab().keys())

           self.assertEqual(vocab_keys[0]   ,					'''<s>''')
           self.assertEqual(vocab_keys[1]   ,					'''<pad>''')
           self.assertEqual(vocab_keys[-1]   ,					'''j''')
           self.assertEqual(len(UpperCAmelCase__)   ,					1_001)


    def 			SCREAMING_SNAKE_CASE				(     self :      List[Any])		->List[str]:





           '''simple docstring'''

           self.assertEqual(self.get_tokenizer().vocab_size   ,					1_001)


    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->List[str]:





           '''simple docstring'''

           A__ 					=       SpeechaTextTokenizer.from_pretrained(self.tmpdirname)

           A__ 					=       tokenizer.tokenize('''This is a test''')
           self.assertListEqual(UpperCAmelCase__   ,					['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est'''])

           self.assertListEqual(
               tokenizer.convert_tokens_to_ids(UpperCAmelCase__)   ,					[289, 50, 14, 174, 386]   ,					)

           A__ 					=       tokenizer.tokenize('''I was born in 92000, and this is falsé.''')
           self.assertListEqual(
               UpperCAmelCase__   ,					[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''9''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''é''', '''.''']   ,					)
           A__ 					=       tokenizer.convert_tokens_to_ids(UpperCAmelCase__)
           self.assertListEqual(UpperCAmelCase__   ,					[12, 25, 88, 59, 28, 23, 11, 4, 606, 351, 351, 351, 7, 16, 70, 50, 76, 84, 10, 4, 8])

           A__ 					=       tokenizer.convert_ids_to_tokens(UpperCAmelCase__)
           self.assertListEqual(
               UpperCAmelCase__   ,					[SPIECE_UNDERLINE + '''I''', SPIECE_UNDERLINE + '''was''', SPIECE_UNDERLINE + '''b''', '''or''', '''n''', SPIECE_UNDERLINE + '''in''', SPIECE_UNDERLINE + '''''', '''<unk>''', '''2''', '''0''', '''0''', '''0''', ''',''', SPIECE_UNDERLINE + '''and''', SPIECE_UNDERLINE + '''this''', SPIECE_UNDERLINE + '''is''', SPIECE_UNDERLINE + '''f''', '''al''', '''s''', '''<unk>''', '''.''']   ,					)





    @slow
    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->List[Any]:





           '''simple docstring'''

           A__ 					=       {'''input_ids''': [[3_791, 797, 31, 11, 64, 797, 31, 2_429, 433, 12, 1_176, 12, 20, 786, 915, 142, 2_413, 240, 37, 3_238, 797, 31, 11, 35, 93, 915, 142, 2_413, 240, 37, 5_540, 567, 1_276, 93, 37, 610, 40, 62, 455, 657, 1_042, 123, 780, 177, 37, 309, 241, 1_298, 514, 20, 292, 2_737, 114, 2_469, 241, 85, 64, 302, 548, 528, 423, 4, 509, 406, 423, 37, 601, 4, 777, 302, 548, 528, 423, 284, 4, 3_388, 511, 459, 4, 3_555, 40, 321, 302, 705, 4, 3_388, 511, 583, 326, 5, 5, 5, 62, 3_310, 560, 177, 2_680, 217, 1_508, 32, 31, 853, 418, 64, 583, 511, 1_605, 62, 35, 93, 560, 177, 2_680, 217, 1_508, 1_521, 64, 583, 511, 519, 62, 20, 1_515, 764, 20, 149, 261, 5_625, 7_972, 20, 5_540, 567, 1_276, 93, 3_925, 1_675, 11, 15, 802, 7_972, 576, 217, 1_508, 11, 35, 93, 1_253, 2_441, 15, 289, 652, 31, 416, 321, 3_842, 115, 40, 911, 8, 476, 619, 4, 380, 142, 423, 335, 240, 35, 93, 264, 8, 11, 335, 569, 420, 163, 5, 2], [260, 548, 528, 423, 20, 451, 20, 2_681, 1_153, 3_434, 20, 5_540, 37, 567, 126, 1_253, 2_441, 3_376, 449, 210, 431, 1_563, 177, 767, 5_540, 11, 1_203, 472, 11, 2_953, 685, 285, 364, 706, 1_153, 20, 6_799, 20, 2_869, 20, 4_464, 126, 40, 2_429, 20, 1_040, 866, 2_664, 418, 20, 318, 20, 1_726, 186, 20, 265, 522, 35, 93, 2_191, 4_634, 20, 1_040, 12, 6_799, 15, 228, 2_356, 142, 31, 11, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [2_575, 2_666, 684, 1_582, 1_176, 12, 627, 149, 619, 20, 4_902, 563, 11, 20, 149, 261, 3_420, 2_356, 174, 142, 4_714, 131, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], '''attention_mask''': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
           # fmt: on

           self.tokenizer_integration_test_util(
               expected_encoding=UpperCAmelCase__   ,					model_name='''facebook/s2t-small-mustc-en-de-st'''   ,					revision='''a14f04cf0776c02f62a8cb800cf7909e15ea23ad'''   ,					)


@require_sentencepiece
class 							UpperCamelCase_       (					unittest.TestCase	):

    '''simple docstring'''






    UpperCAmelCase__				    = '''valhalla/s2t_mustc_multilinguial_medium'''

    UpperCAmelCase__				    = '''C\'est trop cool'''
    UpperCAmelCase__				    = '''Esto es genial'''


    @classmethod
    def 			SCREAMING_SNAKE_CASE				(     cls :      Dict)		->Dict:





           '''simple docstring'''

           A__ 					=       SpeechaTextTokenizer.from_pretrained(cls.checkpoint_name)
           return cls


    def 			SCREAMING_SNAKE_CASE				(     self :      str)		->Dict:





           '''simple docstring'''

           self.assertEqual(self.tokenizer.lang_code_to_id['''pt''']   ,					4)
           self.assertEqual(self.tokenizer.lang_code_to_id['''ru''']   ,					6)
           self.assertEqual(self.tokenizer.lang_code_to_id['''it''']   ,					9)
           self.assertEqual(self.tokenizer.lang_code_to_id['''de''']   ,					11)


    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->Dict:





           '''simple docstring'''

           self.assertEqual(self.tokenizer.vocab_size   ,					10_000)


    def 			SCREAMING_SNAKE_CASE				(     self :      Dict)		->Union[str, Any]:





           '''simple docstring'''

           self.assertIn(UpperCAmelCase__   ,					self.tokenizer.all_special_ids)
           A__ 					=       [ES_CODE, 4, 1_601, 47, 7_647, 2]
           A__ 					=       self.tokenizer.decode(UpperCAmelCase__   ,					skip_special_tokens=UpperCAmelCase__)
           A__ 					=       self.tokenizer.decode(generated_ids[1:]   ,					skip_special_tokens=UpperCAmelCase__)
           self.assertEqual(UpperCAmelCase__   ,					UpperCAmelCase__)
           self.assertNotIn(self.tokenizer.eos_token   ,					UpperCAmelCase__)


    def 			SCREAMING_SNAKE_CASE				(     self :      int)		->str:





           '''simple docstring'''

           A__ 					=       '''fr'''
           A__ 					=       self.tokenizer(self.french_text).input_ids
           self.assertEqual(encoded[0]   ,					UpperCAmelCase__)
           self.assertEqual(encoded[-1]   ,					self.tokenizer.eos_token_id)





    def 			SCREAMING_SNAKE_CASE				(     self :      Optional[int])		->int:





           '''simple docstring'''

           A__ 					=       '''fr'''
           self.assertListEqual(self.tokenizer.prefix_tokens   ,					[FR_CODE])

           A__ 					=       '''es'''
           self.assertListEqual(self.tokenizer.prefix_tokens   ,					[ES_CODE])


