

'''simple docstring'''






import shutil
import tempfile
import unittest

import pytest

from transformers import WhisperTokenizer, is_speech_available
from transformers.testing_utils import require_sentencepiece, require_torch, require_torchaudio

from .test_feature_extraction_whisper import floats_list


if is_speech_available():
      from transformers import WhisperFeatureExtractor, WhisperProcessor


snake_case_       =       5_0_3_5_8
snake_case_       =       5_0_3_6_2
@require_torch
@require_torchaudio
@require_sentencepiece
class 	SCREAMING_SNAKE_CASE__			(				unittest.TestCase      ):




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			Dict     =       "openai/whisper-small.en"
          SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       tempfile.mkdtemp()




    def   __lowerCamelCase      (					self				,     **lowercase__			):






          """simple docstring"""






          return WhisperTokenizer.from_pretrained(self.checkpoint				,     **lowercase__			)




    def   __lowerCamelCase      (					self				,     **lowercase__			):






          """simple docstring"""






          return WhisperFeatureExtractor.from_pretrained(self.checkpoint				,     **lowercase__			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          shutil.rmtree(self.tmpdirname			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			List[str]     =       self.get_tokenizer()
          SCREAMING_SNAKE_CASE_      :			int     =       self.get_feature_extractor()

          SCREAMING_SNAKE_CASE_      :			Any     =       WhisperProcessor(tokenizer=lowercase__				,     feature_extractor=lowercase__			)

          processor.save_pretrained(self.tmpdirname			)
          SCREAMING_SNAKE_CASE_      :			Optional[int]     =       WhisperProcessor.from_pretrained(self.tmpdirname			)

          self.assertEqual(processor.tokenizer.get_vocab()				,     tokenizer.get_vocab()			)
          self.assertIsInstance(processor.tokenizer				,     lowercase__			)

          self.assertEqual(processor.feature_extractor.to_json_string()				,     feature_extractor.to_json_string()			)
          self.assertIsInstance(processor.feature_extractor				,     lowercase__			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       WhisperProcessor(tokenizer=self.get_tokenizer()				,     feature_extractor=self.get_feature_extractor()			)
          processor.save_pretrained(self.tmpdirname			)

          SCREAMING_SNAKE_CASE_      :			Tuple     =       self.get_tokenizer(bos_token="(BOS)"				,     eos_token="(EOS)"			)
          SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       self.get_feature_extractor(do_normalize=lowercase__				,     padding_value=1.0			)

          SCREAMING_SNAKE_CASE_      :			List[Any]     =       WhisperProcessor.from_pretrained(
              self.tmpdirname				,     bos_token="(BOS)"				,     eos_token="(EOS)"				,     do_normalize=lowercase__				,     padding_value=1.0			)

          self.assertEqual(processor.tokenizer.get_vocab()				,     tokenizer_add_kwargs.get_vocab()			)
          self.assertIsInstance(processor.tokenizer				,     lowercase__			)

          self.assertEqual(processor.feature_extractor.to_json_string()				,     feature_extractor_add_kwargs.to_json_string()			)
          self.assertIsInstance(processor.feature_extractor				,     lowercase__			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       self.get_feature_extractor()
          SCREAMING_SNAKE_CASE_      :			Tuple     =       self.get_tokenizer()

          SCREAMING_SNAKE_CASE_      :			Optional[int]     =       WhisperProcessor(tokenizer=lowercase__				,     feature_extractor=lowercase__			)

          SCREAMING_SNAKE_CASE_      :			List[Any]     =       floats_list((3, 1000)			)

          SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       feature_extractor(lowercase__				,     return_tensors="np"			)
          SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       processor(lowercase__				,     return_tensors="np"			)

          for key in input_feat_extract.keys():
                self.assertAlmostEqual(input_feat_extract[key].sum()				,     input_processor[key].sum()				,     delta=1e-2			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			int     =       self.get_feature_extractor()
          SCREAMING_SNAKE_CASE_      :			Optional[int]     =       self.get_tokenizer()

          SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       WhisperProcessor(tokenizer=lowercase__				,     feature_extractor=lowercase__			)

          SCREAMING_SNAKE_CASE_      :			int     =       "This is a test string"

          SCREAMING_SNAKE_CASE_      :			Any     =       processor(text=lowercase__			)

          SCREAMING_SNAKE_CASE_      :			Optional[int]     =       tokenizer(lowercase__			)

          for key in encoded_tok.keys():
                self.assertListEqual(encoded_tok[key]				,     encoded_processor[key]			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			int     =       self.get_feature_extractor()
          SCREAMING_SNAKE_CASE_      :			int     =       self.get_tokenizer()

          SCREAMING_SNAKE_CASE_      :			str     =       WhisperProcessor(tokenizer=lowercase__				,     feature_extractor=lowercase__			)

          SCREAMING_SNAKE_CASE_      :			int     =       [[1, 4, 5, 8, 1, 0, 8], [3, 4, 3, 1, 1, 8, 9]]

          SCREAMING_SNAKE_CASE_      :			List[Any]     =       processor.batch_decode(lowercase__			)
          SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       tokenizer.batch_decode(lowercase__			)

          self.assertListEqual(lowercase__				,     lowercase__			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       self.get_feature_extractor()
          SCREAMING_SNAKE_CASE_      :			List[str]     =       self.get_tokenizer()

          SCREAMING_SNAKE_CASE_      :			List[str]     =       WhisperProcessor(tokenizer=lowercase__				,     feature_extractor=lowercase__			)

          self.assertListEqual(
              processor.model_input_names				,     feature_extractor.model_input_names				,     msg="`processor` and `feature_extractor` model input names do not match"				,     )




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			Tuple     =       self.get_feature_extractor()
          SCREAMING_SNAKE_CASE_      :			str     =       self.get_tokenizer()

          SCREAMING_SNAKE_CASE_      :			Any     =       WhisperProcessor(tokenizer=lowercase__				,     feature_extractor=lowercase__			)
          SCREAMING_SNAKE_CASE_      :			List[Any]     =       processor.get_decoder_prompt_ids(task="transcribe"				,     no_timestamps=lowercase__			)

          self.assertIsInstance(lowercase__				,     lowercase__			)
          for ids in forced_decoder_ids:
                self.assertIsInstance(lowercase__				,     (list, tuple)			)

          SCREAMING_SNAKE_CASE_      :			List[Any]     =       [TRANSCRIBE, NOTIMESTAMPS]
          self.assertListEqual([ids[-1] for ids in forced_decoder_ids]				,     lowercase__			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			int     =       WhisperProcessor(tokenizer=self.get_tokenizer()				,     feature_extractor=self.get_feature_extractor()			)
          SCREAMING_SNAKE_CASE_      :			Optional[int]     =       processor.get_prompt_ids("Mr. Quilter"			)
          SCREAMING_SNAKE_CASE_      :			int     =       processor.tokenizer.decode(lowercase__			)

          self.assertListEqual(prompt_ids.tolist()				,     [5_0360, 1770, 13, 2264, 346, 353]			)
          self.assertEqual(lowercase__				,     "<|startofprev|> Mr. Quilter"			)




    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			int     =       WhisperProcessor(tokenizer=self.get_tokenizer()				,     feature_extractor=self.get_feature_extractor()			)
          SCREAMING_SNAKE_CASE_      :			Optional[int]     =       processor.get_prompt_ids(""			)
          SCREAMING_SNAKE_CASE_      :			Union[str, Any]     =       processor.tokenizer.decode(lowercase__			)

          self.assertListEqual(prompt_ids.tolist()				,     [5_0360, 220]			)
          self.assertEqual(lowercase__				,     "<|startofprev|> "			)







    def   __lowerCamelCase      (					self			):






          """simple docstring"""






          SCREAMING_SNAKE_CASE_      :			Tuple     =       WhisperProcessor(tokenizer=self.get_tokenizer()				,     feature_extractor=self.get_feature_extractor()			)

          def _test_prompt_error_raised_helper(lowercase__				,     lowercase__			):
                with pytest.raises(lowercase__			) as excinfo:
                      processor.get_prompt_ids(lowercase__			)
                SCREAMING_SNAKE_CASE_      :			Optional[Any]     =       F"Encountered text in the prompt corresponding to disallowed special token: {special_token}."
                self.assertEqual(lowercase__				,     str(excinfo.value			)			)

          _test_prompt_error_raised_helper("<|startofprev|> test"				,     "<|startofprev|>"			)
          _test_prompt_error_raised_helper("test <|notimestamps|>"				,     "<|notimestamps|>"			)
          _test_prompt_error_raised_helper("test <|zh|> test <|transcribe|>"				,     "<|zh|>"			)







import argparse
import collections
import importlib.util
import os
import re


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/check_table.py
UpperCamelCase__              =						'src/diffusers'
UpperCamelCase__              =						'docs/source/en'
UpperCamelCase__              =						'.'






def 					snake_case					(							A						,   A						,   A			)							->							int:




	'''simple docstring'''


	with open(A						,   'r'						,   encoding='utf-8'						,   newline='\n'			) as f:
		lowerCamelCase_							:		int							   =							f.readlines()
	# Find the start prompt.
	lowerCamelCase_							:		Tuple							   =							0
	while not lines[start_index].startswith(A			):
		start_index += 1
	start_index += 1

	lowerCamelCase_							:		Tuple							   =							start_index
	while not lines[end_index].startswith(A			):
		end_index += 1
	end_index -= 1

	while len(lines[start_index]			) <= 1:
		start_index += 1
	while len(lines[end_index]			) <= 1:
		end_index -= 1
	end_index += 1
	return "".join(lines[start_index:end_index]			), start_index, end_index, lines


# Add here suffixes that are used to identify models, separated by |
UpperCamelCase__              =						'Model|Encoder|Decoder|ForConditionalGeneration'
# Regexes that match TF/Flax/PT model names.
UpperCamelCase__              =						re.compile(r'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
UpperCamelCase__              =						re.compile(r'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
UpperCamelCase__              =						re.compile(r'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# This is to make sure the diffusers module imported is the one in the repo.
UpperCamelCase__              =						importlib.util.spec_from_file_location(
    'diffusers',
    os.path.join(TRANSFORMERS_PATH, '__init__.py'),
    submodule_search_locations=[TRANSFORMERS_PATH],
)
UpperCamelCase__              =						spec.loader.load_module()






def 					snake_case					(							A			)							->							str:




	'''simple docstring'''


	lowerCamelCase_							:		Dict							   =							re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'						,   A			)
	return [m.group(0			) for m in matches]






def 					snake_case					(							A						,   A			)							->							List[Any]:




	'''simple docstring'''


	lowerCamelCase_							:		str							   =							2 if text == '✅' or text == '❌' else len(A			)
	lowerCamelCase_							:		List[str]							   =							(width - text_length) // 2
	lowerCamelCase_							:		int							   =							width - text_length - left_indent
	return " " * left_indent + text + " " * right_indent






def 					snake_case					(							)							->							Dict:




	'''simple docstring'''


	lowerCamelCase_							:		Union[str, Any]							   =							diffusers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
	lowerCamelCase_							:		Optional[Any]							   =							{
	    name: config_mapping_names[code]
	    for code, name in diffusers_module.MODEL_NAMES_MAPPING.items()
	    if code in config_mapping_names
	}
	lowerCamelCase_							:		Optional[int]							   =							{name: config.replace('ConfigMixin'						,   ''			) for name, config in model_name_to_config.items()}

	# Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax.
	lowerCamelCase_							:		Tuple							   =							collections.defaultdict(A			)
	lowerCamelCase_							:		Any							   =							collections.defaultdict(A			)
	lowerCamelCase_							:		int							   =							collections.defaultdict(A			)
	lowerCamelCase_							:		Optional[int]							   =							collections.defaultdict(A			)
	lowerCamelCase_							:		Dict							   =							collections.defaultdict(A			)

	# Let's lookup through all diffusers object (once).
	for attr_name in dir(A			):
		lowerCamelCase_							:		Tuple							   =							None
		if attr_name.endswith('Tokenizer'			):
			lowerCamelCase_							:		Union[str, Any]							   =							slow_tokenizers
			lowerCamelCase_							:		Optional[Any]							   =							attr_name[:-9]
		elif attr_name.endswith('TokenizerFast'			):
			lowerCamelCase_							:		Any							   =							fast_tokenizers
			lowerCamelCase_							:		Optional[Any]							   =							attr_name[:-13]
		elif _re_tf_models.match(A			) is not None:
			lowerCamelCase_							:		Tuple							   =							tf_models
			lowerCamelCase_							:		Optional[int]							   =							_re_tf_models.match(A			).groups()[0]
		elif _re_flax_models.match(A			) is not None:
			lowerCamelCase_							:		Union[str, Any]							   =							flax_models
			lowerCamelCase_							:		Union[str, Any]							   =							_re_flax_models.match(A			).groups()[0]
		elif _re_pt_models.match(A			) is not None:
			lowerCamelCase_							:		Union[str, Any]							   =							pt_models
			lowerCamelCase_							:		Optional[int]							   =							_re_pt_models.match(A			).groups()[0]

		if lookup_dict is not None:
			while len(A			) > 0:
				if attr_name in model_name_to_prefix.values():
					lowerCamelCase_							:		List[Any]							   =							True
					break
				# Try again after removing the last word in the name
				lowerCamelCase_							:		int							   =							''.join(camel_case_split(A			)[:-1]			)

    # Let's build that table!
	lowerCamelCase_							:		Optional[int]							   =							list(model_name_to_config.keys()			)
	model_names.sort(key=str.lower			)
	lowerCamelCase_							:		str							   =							['Model', 'Tokenizer slow', 'Tokenizer fast', 'PyTorch support', 'TensorFlow support', 'Flax Support']
	# We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side).
	lowerCamelCase_							:		str							   =							[len(A			) + 2 for c in columns]
	lowerCamelCase_							:		Optional[int]							   =							max([len(A			) for name in model_names]			) + 2

	# Build the table per se
	lowerCamelCase_							:		Optional[Any]							   =							'|' + '|'.join([_center_text(A						,   A			) for c, w in zip(A						,   A			)]			) + '|\n'
	# Use ":-----:" format to center-aligned table cell texts
	table += "|" + "|".join([':' + '-' * (w - 2) + ':' for w in widths]			) + "|\n"

	lowerCamelCase_							:		str							   =							{True: '✅', False: '❌'}
	for name in model_names:
		lowerCamelCase_							:		List[str]							   =							model_name_to_prefix[name]
		lowerCamelCase_							:		Tuple							   =							[
		    name,
		    check[slow_tokenizers[prefix]],
		    check[fast_tokenizers[prefix]],
		    check[pt_models[prefix]],
		    check[tf_models[prefix]],
		    check[flax_models[prefix]],
		]
		table += "|" + "|".join([_center_text(A						,   A			) for l, w in zip(A						,   A			)]			) + "|\n"
	return table






def 					snake_case					(							A=False			)							->							List[Any]:




	'''simple docstring'''


	lowerCamelCase_   ,			lowerCamelCase_   ,			lowerCamelCase_   ,			lowerCamelCase_							:		Tuple							   =							_find_text_in_file(
	    filename=os.path.join(A						,   'index.mdx'			)						,   start_prompt='<!--This table is updated automatically from the auto modules'						,   end_prompt='<!-- End table-->'						,   )
	lowerCamelCase_							:		Any							   =							get_model_table_from_auto_modules()

	if current_table != new_table:
		if overwrite:
			with open(os.path.join(A						,   'index.mdx'			)						,   'w'						,   encoding='utf-8'						,   newline='\n'			) as f:
				f.writelines(lines[:start_index] + [new_table] + lines[end_index:]			)
		else:
			raise ValueError(
			    'The model table in the `index.mdx` has not been updated. Run `make fix-copies` to fix this.'			)


if __name__ == "__main__":
							UpperCamelCase__              =						argparse.ArgumentParser()
							parser.add_argument('--fix_and_overwrite', action='store_true', help='Whether to fix inconsistencies.')
							UpperCamelCase__              =						parser.parse_args()

							check_model_table(args.fix_and_overwrite)
