

import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
A									=  'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
A									=  direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
A									=  re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
A									=  re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
A									=  re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
A									=  [
    ('pretraining', 'MODEL_FOR_PRETRAINING_MAPPING_NAMES', 'AutoModelForPreTraining'),
    ('feature-extraction', 'MODEL_MAPPING_NAMES', 'AutoModel'),
    ('audio-classification', 'MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForAudioClassification'),
    ('text-generation', 'MODEL_FOR_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForCausalLM'),
    ('automatic-speech-recognition', 'MODEL_FOR_CTC_MAPPING_NAMES', 'AutoModelForCTC'),
    ('image-classification', 'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForImageClassification'),
    ('image-segmentation', 'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES', 'AutoModelForImageSegmentation'),
    ('fill-mask', 'MODEL_FOR_MASKED_LM_MAPPING_NAMES', 'AutoModelForMaskedLM'),
    ('object-detection', 'MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES', 'AutoModelForObjectDetection'),
    (
        'zero-shot-object-detection',
        'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES',
        'AutoModelForZeroShotObjectDetection',
    ),
    ('question-answering', 'MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES', 'AutoModelForQuestionAnswering'),
    ('text2text-generation', 'MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForSeq2SeqLM'),
    ('text-classification', 'MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForSequenceClassification'),
    ('automatic-speech-recognition', 'MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES', 'AutoModelForSpeechSeq2Seq'),
    (
        'table-question-answering',
        'MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForTableQuestionAnswering',
    ),
    ('token-classification', 'MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForTokenClassification'),
    ('multiple-choice', 'MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES', 'AutoModelForMultipleChoice'),
    (
        'next-sentence-prediction',
        'MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES',
        'AutoModelForNextSentencePrediction',
    ),
    (
        'audio-frame-classification',
        'MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForAudioFrameClassification',
    ),
    ('audio-xvector', 'MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES', 'AutoModelForAudioXVector'),
    (
        'document-question-answering',
        'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForDocumentQuestionAnswering',
    ),
    (
        'visual-question-answering',
        'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForVisualQuestionAnswering',
    ),
    ('image-to-text', 'MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES', 'AutoModelForVision2Seq'),
    (
        'zero-shot-image-classification',
        'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForZeroShotImageClassification',
    ),
    ('depth-estimation', 'MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES', 'AutoModelForDepthEstimation'),
    ('video-classification', 'MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForVideoClassification'),
    ('mask-generation', 'MODEL_FOR_MASK_GENERATION_MAPPING_NAMES', 'AutoModelForMaskGeneration'),
]





def a(lowercase__						):





	'''simple docstring'''
	snake_case_   		=			re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'      ,			lowercase__						)
	return [m.group(0						) for m in matches]





def a():





	'''simple docstring'''
	snake_case_   		=			transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
	snake_case_   		=			{
	    config.replace('Config'      ,			''						): model_type for model_type, config in config_maping_names.items()
	}

	# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
	snake_case_   		=			collections.defaultdict(lowercase__						)
	snake_case_   		=			collections.defaultdict(lowercase__						)
	snake_case_   		=			collections.defaultdict(lowercase__						)

	# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
	for attr_name in dir(lowercase__						):
		snake_case_   		=			None
		if _re_tf_models.match(lowercase__						) is not None:
			snake_case_   		=			tf_models
			snake_case_   		=			_re_tf_models.match(lowercase__						).groups()[0]
		elif _re_flax_models.match(lowercase__						) is not None:
			snake_case_   		=			flax_models
			snake_case_   		=			_re_flax_models.match(lowercase__						).groups()[0]
		elif _re_pt_models.match(lowercase__						) is not None:
			snake_case_   		=			pt_models
			snake_case_   		=			_re_pt_models.match(lowercase__						).groups()[0]

		if lookup_dict is not None:
			while len(lowercase__						) > 0:
				if attr_name in model_prefix_to_model_type:
					snake_case_   		=			True
					break
				# Try again after removing the last word in the name
				snake_case_   		=			''.join(camel_case_split(lowercase__						)[:-1]						)

	snake_case_   		=			set(list(pt_models.keys()						) + list(tf_models.keys()						) + list(flax_models.keys()						)						)
	snake_case_   		=			list(lowercase__						)
	all_models.sort()

	snake_case_   		=			{'model_type': all_models}
	snake_case_   		=			[pt_models[t] for t in all_models]
	snake_case_   		=			[tf_models[t] for t in all_models]
	snake_case_   		=			[flax_models[t] for t in all_models]

	# Now let's use the auto-mapping names to make sure
	snake_case_   		=			{}
	for t in all_models:
		if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
			snake_case_   		=			'AutoProcessor'
		elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
			snake_case_   		=			'AutoTokenizer'
		elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
			snake_case_   		=			'AutoFeatureExtractor'
		else:
			# Default to AutoTokenizer if a model has nothing, for backward compatibility.
			snake_case_   		=			'AutoTokenizer'

	snake_case_   		=			[processors[t] for t in all_models]

	return pd.DataFrame(lowercase__						)





def a(lowercase__						):





	'''simple docstring'''
	snake_case_   		=			[
	    transformers_module.models.auto.modeling_auto,
	    transformers_module.models.auto.modeling_tf_auto,
	    transformers_module.models.auto.modeling_flax_auto,
	]
	for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
		snake_case_   		=			[model_mapping, f"""TF_{model_mapping}""", f"""FLAX_{model_mapping}"""]
		snake_case_   		=			[auto_class, f"""TF_{auto_class}""", f"""Flax_{auto_class}"""]
		# Loop through all three frameworks
		for module, cls, mapping in zip(lowercase__      ,			lowercase__      ,			lowercase__						):
			# The type of pipeline may not exist in this framework
			if not hasattr(lowercase__      ,			lowercase__						):
				continue
			# First extract all model_names
			snake_case_   		=			[]
			for name in getattr(lowercase__      ,			lowercase__						).values():
				if isinstance(lowercase__      ,			lowercase__						):
					model_names.append(lowercase__						)
				else:
					model_names.extend(list(lowercase__						)						)

            # Add pipeline tag and auto model class for those models
			table.update({model_name: (pipeline_tag, cls) for model_name in model_names}						)

	return table





def a(lowercase__      ,			lowercase__						):





	'''simple docstring'''
	snake_case_   		=			get_frameworks_table()
	snake_case_   		=			Dataset.from_pandas(lowercase__						)

	snake_case_   		=			hf_hub_download(
	    'huggingface/transformers-metadata'      ,			'pipeline_tags.json'      ,			repo_type='dataset'      ,			token=lowercase__						)
	snake_case_   		=			Dataset.from_json(lowercase__						)
	snake_case_   		=			{
	    tags_dataset[i]['model_class']: (tags_dataset[i]['pipeline_tag'], tags_dataset[i]['auto_class'])
	    for i in range(len(lowercase__						)						)
	}
	snake_case_   		=			update_pipeline_and_auto_class_table(lowercase__						)

	# Sort the model classes to avoid some nondeterministic updates to create false update commits.
	snake_case_   		=			sorted(table.keys()						)
	snake_case_   		=			pd.DataFrame(
	    {
	        'model_class': model_classes,
	        'pipeline_tag': [table[m][0] for m in model_classes],
	        'auto_class': [table[m][1] for m in model_classes],
	    }						)
	snake_case_   		=			Dataset.from_pandas(lowercase__						)

	with tempfile.TemporaryDirectory() as tmp_dir:
		frameworks_dataset.to_json(os.path.join(lowercase__      ,			'frameworks.json'						)						)
		tags_dataset.to_json(os.path.join(lowercase__      ,			'pipeline_tags.json'						)						)

		if commit_sha is not None:
			snake_case_   		=			(
			    f"""Update with commit {commit_sha}\n\nSee: """
			    f"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
			)
		else:
			snake_case_   		=			'Update'

		upload_folder(
		    repo_id='huggingface/transformers-metadata'      ,			folder_path=lowercase__      ,			repo_type='dataset'      ,			token=lowercase__      ,			commit_message=lowercase__      ,			)





def a():





	'''simple docstring'''
	snake_case_   		=			{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
	snake_case_   		=			transformers_module.pipelines.SUPPORTED_TASKS
	snake_case_   		=			[]
	for key in pipeline_tasks:
		if key not in in_table:
			snake_case_   		=			pipeline_tasks[key]['pt']
			if isinstance(lowercase__      ,			(list, tuple)						):
				snake_case_   		=			model[0]
			snake_case_   		=			model.__name__
			if model not in in_table.values():
				missing.append(lowercase__						)

	if len(lowercase__						) > 0:
		snake_case_   		=			', '.join(lowercase__						)
		raise ValueError(
		    'The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '
		    f"""`utils/update_metadata.py`: {msg}. Please add them!"""						)


if __name__ == "__main__":
		A									=  argparse.ArgumentParser()
		parser.add_argument('--token', type=str, help='The token to use to push to the transformers-metadata dataset.')
		parser.add_argument('--commit_sha', type=str, help='The sha of the commit going with this update.')
		parser.add_argument('--check-only', action='store_true', help='Activate to just check all pipelines are present.')
		A									=  parser.parse_args()

		if args.check_only:
				check_pipeline_tags()
		else:
				update_metadata(args.token, args.commit_sha)



import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ASTConfig
from transformers.testing_utils import require_torch, require_torchaudio, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_torchaudio_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch
		from torch import nn

		from transformers import ASTForAudioClassification, ASTModel
		from transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer import (
		    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
		)


if is_torchaudio_available():
		import torchaudio

		from transformers import ASTFeatureExtractor


class SCREAMING_SNAKE_CASE	:






			"""simple docstring"""




			def __init__(     self			,					__UpperCamelCase			,					__UpperCamelCase=13			,					__UpperCamelCase=2			,					__UpperCamelCase=24			,					__UpperCamelCase=16			,					__UpperCamelCase=True			,					__UpperCamelCase=True			,					__UpperCamelCase=32			,					__UpperCamelCase=5			,					__UpperCamelCase=4			,					__UpperCamelCase=37			,					__UpperCamelCase="gelu"			,					__UpperCamelCase=0.1			,					__UpperCamelCase=0.1			,					__UpperCamelCase=10			,					__UpperCamelCase=0.02			,					__UpperCamelCase=None			,					__UpperCamelCase=2			,					__UpperCamelCase=2			,					):

				"""simple docstring"""


				snake_case_   		=			parent
				snake_case_   		=			batch_size
				snake_case_   		=			patch_size
				snake_case_   		=			max_length
				snake_case_   		=			num_mel_bins
				snake_case_   		=			is_training
				snake_case_   		=			use_labels
				snake_case_   		=			hidden_size
				snake_case_   		=			num_hidden_layers
				snake_case_   		=			num_attention_heads
				snake_case_   		=			intermediate_size
				snake_case_   		=			hidden_act
				snake_case_   		=			hidden_dropout_prob
				snake_case_   		=			attention_probs_dropout_prob
				snake_case_   		=			type_sequence_label_size
				snake_case_   		=			initializer_range
				snake_case_   		=			scope
				snake_case_   		=			frequency_stride
				snake_case_   		=			time_stride

				# in AST, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
				snake_case_   		=			(self.num_mel_bins - self.patch_size) // self.frequency_stride + 1
				snake_case_   		=			(self.max_length - self.patch_size) // self.time_stride + 1
				snake_case_   		=			frequency_out_dimension * time_out_dimension
				snake_case_   		=			num_patches + 2


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			floats_tensor([self.batch_size, self.max_length, self.num_mel_bins]				)

				snake_case_   		=			None
				if self.use_labels:
					snake_case_   		=			ids_tensor([self.batch_size]			,					self.type_sequence_label_size				)

				snake_case_   		=			self.get_config()

				return config, input_values, labels


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				return ASTConfig(
				    patch_size=self.patch_size			,					max_length=self.max_length			,					num_mel_bins=self.num_mel_bins			,					hidden_size=self.hidden_size			,					num_hidden_layers=self.num_hidden_layers			,					num_attention_heads=self.num_attention_heads			,					intermediate_size=self.intermediate_size			,					hidden_act=self.hidden_act			,					hidden_dropout_prob=self.hidden_dropout_prob			,					attention_probs_dropout_prob=self.attention_probs_dropout_prob			,					is_decoder=__UpperCamelCase			,					initializer_range=self.initializer_range			,					frequency_stride=self.frequency_stride			,					time_stride=self.time_stride			,					)


			def       __lowerCAmelCase			(     self			,					__UpperCamelCase			,					__UpperCamelCase			,					__UpperCamelCase				):

				"""simple docstring"""


				snake_case_   		=			ASTModel(config=__UpperCamelCase				)
				model.to(__UpperCamelCase				)
				model.eval()
				snake_case_   		=			model(__UpperCamelCase				)
				self.parent.assertEqual(result.last_hidden_state.shape			,					(self.batch_size, self.seq_length, self.hidden_size)				)







			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			self.prepare_config_and_inputs()
				(
				    (
				    snake_case_
				)		,							(
				    snake_case_
				)		,							(
				    snake_case_
				)		,							
				)   		=			config_and_inputs
				snake_case_   		=			{'input_values': input_values}
				return config, inputs_dict







@require_torch
class SCREAMING_SNAKE_CASE	(		__snake_case				,    __snake_case				,    unittest.TestCase				):






			"""simple docstring"""




			__A    	=	(
			    (
			        ASTModel,
			        ASTForAudioClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			__A    	=	(
			    {"""audio-classification""": ASTForAudioClassification, """feature-extraction""": ASTModel}
			    if is_torch_available()
			    else {}
			)
			__A    	=	False
			__A    	=	False
			__A    	=	False
			__A    	=	False


			def       __lowerCAmelCase			(     self			,					__UpperCamelCase			,					__UpperCamelCase			,					__UpperCamelCase			,					__UpperCamelCase			,					__UpperCamelCase				):

				"""simple docstring"""


				if pipeline_test_casse_name == "AudioClassificationPipelineTests":
					return True

				return False


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			ASTModelTester(self				)
				snake_case_   		=			ConfigTester(self			,					config_class=__UpperCamelCase			,					has_text_modality=__UpperCamelCase			,					hidden_size=37				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				self.config_tester.run_common_tests()


			@unittest.skip(reason='AST does not use inputs_embeds'				)
			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				pass


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_		,							snake_case_   		=			self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
					snake_case_   		=			model_class(__UpperCamelCase				)
					self.assertIsInstance(model.get_input_embeddings()			,					(nn.Module)				)
					snake_case_   		=			model.get_output_embeddings()
					self.assertTrue(x is None or isinstance(__UpperCamelCase			,					nn.Linear				)				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_		,							snake_case_   		=			self.model_tester.prepare_config_and_inputs_for_common()

				for model_class in self.all_model_classes:
					snake_case_   		=			model_class(__UpperCamelCase				)
					snake_case_   		=			inspect.signature(model.forward				)
					# signature.parameters is an OrderedDict => so arg_names order is deterministic
					snake_case_   		=			[*signature.parameters.keys()]

					snake_case_   		=			['input_values']
					self.assertListEqual(arg_names[:1]			,					__UpperCamelCase				)


			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			self.model_tester.prepare_config_and_inputs()
				self.model_tester.create_and_check_model(*__UpperCamelCase				)







			@slow
			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				for model_name in AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
					snake_case_   		=			ASTModel.from_pretrained(__UpperCamelCase				)
					self.assertIsNotNone(__UpperCamelCase				)







def a():





	'''simple docstring'''
	snake_case_   		=			hf_hub_download(
	    repo_id='nielsr/audio-spectogram-transformer-checkpoint'      ,			filename='sample_audio.flac'      ,			repo_type='dataset'						)

	snake_case_		,							snake_case_   		=			torchaudio.load(lowercase__						)

	return audio, sampling_rate






@require_torch
@require_torchaudio
class SCREAMING_SNAKE_CASE	(		unittest.TestCase				):






			"""simple docstring"""




			@cached_property
			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				return (
				    ASTFeatureExtractor.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593'				)
				    if is_torchaudio_available()
				    else None
				)







			@slow
			def       __lowerCAmelCase			(     self				):

				"""simple docstring"""


				snake_case_   		=			self.default_feature_extractor
				snake_case_   		=			ASTForAudioClassification.from_pretrained('MIT/ast-finetuned-audioset-10-10-0.4593'				).to(__UpperCamelCase				)

				snake_case_   		=			self.default_feature_extractor
				snake_case_		,							snake_case_   		=			prepare_audio()
				snake_case_   		=			audio.squeeze().numpy()
				snake_case_   		=			feature_extractor(__UpperCamelCase			,					sampling_rate=__UpperCamelCase			,					return_tensors='pt'				).to(__UpperCamelCase				)

				# forward pass
				with torch.no_grad():
					snake_case_   		=			model(**__UpperCamelCase				)

				# verify the logits
				snake_case_   		=			torch.Size((1, 5_27)				)
				self.assertEqual(outputs.logits.shape			,					__UpperCamelCase				)

				snake_case_   		=			torch.tensor([-0.8760, -7.0042, -8.6602]				).to(__UpperCamelCase				)

				self.assertTrue(torch.allclose(outputs.logits[0, :3]			,					__UpperCamelCase			,					atol=1E-4				)				)
