





import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
snake_case_							=			'''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
snake_case_							=			direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
snake_case_							=			re.compile(r'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
snake_case_							=			re.compile(r'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
snake_case_							=			re.compile(r'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
snake_case_							=			[
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]


def 					A__	(       SCREAMING_SNAKE_CASE_		)   ->     Dict:
		lowerCamelCase   : str								=re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''							,       SCREAMING_SNAKE_CASE_		)
		return [m.group(0		) for m in matches]


def 					A__	(       )   ->     Dict:
		lowerCamelCase   : Dict								=transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
		lowerCamelCase   : Optional[int]								={
		    config.replace('''Config'''							,       ''''''		): model_type for model_type, config in config_maping_names.items()
		}

		# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
		lowerCamelCase   : List[str]								=collections.defaultdict(SCREAMING_SNAKE_CASE_		)
		lowerCamelCase   : Union[str, Any]								=collections.defaultdict(SCREAMING_SNAKE_CASE_		)
		lowerCamelCase   : Tuple								=collections.defaultdict(SCREAMING_SNAKE_CASE_		)

		# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
		for attr_name in dir(SCREAMING_SNAKE_CASE_		):
				lowerCamelCase   : Union[str, Any]								=None
				if _re_tf_models.match(SCREAMING_SNAKE_CASE_		) is not None:
						lowerCamelCase   : List[Any]								=tf_models
						lowerCamelCase   : Any								=_re_tf_models.match(SCREAMING_SNAKE_CASE_		).groups()[0]
				elif _re_flax_models.match(SCREAMING_SNAKE_CASE_		) is not None:
						lowerCamelCase   : List[Any]								=flax_models
						lowerCamelCase   : Optional[int]								=_re_flax_models.match(SCREAMING_SNAKE_CASE_		).groups()[0]
				elif _re_pt_models.match(SCREAMING_SNAKE_CASE_		) is not None:
						lowerCamelCase   : Union[str, Any]								=pt_models
						lowerCamelCase   : Optional[Any]								=_re_pt_models.match(SCREAMING_SNAKE_CASE_		).groups()[0]

				if lookup_dict is not None:
						while len(SCREAMING_SNAKE_CASE_		) > 0:
								if attr_name in model_prefix_to_model_type:
										lowerCamelCase   : Optional[Any]								=True
										break
								# Try again after removing the last word in the name
								lowerCamelCase   : Optional[int]								=''''''.join(camel_case_split(SCREAMING_SNAKE_CASE_		)[:-1]		)

		lowerCamelCase   : Union[str, Any]								=set(list(pt_models.keys()		) + list(tf_models.keys()		) + list(flax_models.keys()		)		)
		lowerCamelCase   : int								=list(SCREAMING_SNAKE_CASE_		)
		all_models.sort()

		lowerCamelCase   : List[str]								={'''model_type''': all_models}
		lowerCamelCase   : int								=[pt_models[t] for t in all_models]
		lowerCamelCase   : Dict								=[tf_models[t] for t in all_models]
		lowerCamelCase   : Optional[int]								=[flax_models[t] for t in all_models]

		# Now let's use the auto-mapping names to make sure
		lowerCamelCase   : Tuple								={}
		for t in all_models:
				if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
						lowerCamelCase   : List[str]								='''AutoProcessor'''
				elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
						lowerCamelCase   : Union[str, Any]								='''AutoTokenizer'''
				elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
						lowerCamelCase   : int								='''AutoFeatureExtractor'''
				else:
						# Default to AutoTokenizer if a model has nothing, for backward compatibility.
						lowerCamelCase   : Dict								='''AutoTokenizer'''

		lowerCamelCase   : List[Any]								=[processors[t] for t in all_models]

		return pd.DataFrame(SCREAMING_SNAKE_CASE_		)


def 					A__	(       SCREAMING_SNAKE_CASE_		)   ->     Union[str, Any]:
		lowerCamelCase   : int								=[
		    transformers_module.models.auto.modeling_auto,
		    transformers_module.models.auto.modeling_tf_auto,
		    transformers_module.models.auto.modeling_flax_auto,
		]
		for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
				lowerCamelCase   : List[str]								=[model_mapping, F"TF_{model_mapping}", F"FLAX_{model_mapping}"]
				lowerCamelCase   : Optional[Any]								=[auto_class, F"TF_{auto_class}", F"Flax_{auto_class}"]
				# Loop through all three frameworks
				for module, cls, mapping in zip(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		):
						# The type of pipeline may not exist in this framework
						if not hasattr(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		):
								continue
						# First extract all model_names
						lowerCamelCase   : Union[str, Any]								=[]
						for name in getattr(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		).values():
								if isinstance(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		):
										model_names.append(SCREAMING_SNAKE_CASE_		)
								else:
										model_names.extend(list(SCREAMING_SNAKE_CASE_		)		)

            # Add pipeline tag and auto model class for those models
						table.update({model_name: (pipeline_tag, cls) for model_name in model_names}		)

		return table


def 					A__	(       SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		)   ->     Dict:
		lowerCamelCase   : Optional[int]								=get_frameworks_table()
		lowerCamelCase   : Optional[Any]								=Dataset.from_pandas(SCREAMING_SNAKE_CASE_		)

		lowerCamelCase   : Optional[Any]								=hf_hub_download(
		    '''huggingface/transformers-metadata'''							,       '''pipeline_tags.json'''							,       repo_type='''dataset'''							,       token=SCREAMING_SNAKE_CASE_		)
		lowerCamelCase   : int								=Dataset.from_json(SCREAMING_SNAKE_CASE_		)
		lowerCamelCase   : Optional[int]								={
		    tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
		    for i in range(len(SCREAMING_SNAKE_CASE_		)		)
		}
		lowerCamelCase   : Tuple								=update_pipeline_and_auto_class_table(SCREAMING_SNAKE_CASE_		)

		# Sort the model classes to avoid some nondeterministic updates to create false update commits.
		lowerCamelCase   : Union[str, Any]								=sorted(table.keys()		)
		lowerCamelCase   : List[str]								=pd.DataFrame(
		    {
		        '''model_class''': model_classes,
		        '''pipeline_tag''': [table[m][0] for m in model_classes],
		        '''auto_class''': [table[m][1] for m in model_classes],
		    }		)
		lowerCamelCase   : Any								=Dataset.from_pandas(SCREAMING_SNAKE_CASE_		)

		with tempfile.TemporaryDirectory() as tmp_dir:
				frameworks_dataset.to_json(os.path.join(SCREAMING_SNAKE_CASE_							,       '''frameworks.json'''		)		)
				tags_dataset.to_json(os.path.join(SCREAMING_SNAKE_CASE_							,       '''pipeline_tags.json'''		)		)

				if commit_sha is not None:
						lowerCamelCase   : Optional[Any]								=(
						    F"Update with commit {commit_sha}\n\nSee: "
						    F"https://github.com/huggingface/transformers/commit/{commit_sha}"
						)
				else:
						lowerCamelCase   : str								='''Update'''

				upload_folder(
				    repo_id='''huggingface/transformers-metadata'''							,       folder_path=SCREAMING_SNAKE_CASE_							,       repo_type='''dataset'''							,       token=SCREAMING_SNAKE_CASE_							,       commit_message=SCREAMING_SNAKE_CASE_							,       )


def 					A__	(       )   ->     Optional[int]:
		lowerCamelCase   : Union[str, Any]								={tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
		lowerCamelCase   : Dict								=transformers_module.pipelines.SUPPORTED_TASKS
		lowerCamelCase   : Optional[Any]								=[]
		for key in pipeline_tasks:
				if key not in in_table:
						lowerCamelCase   : Optional[int]								=pipeline_tasks[key]['''pt''']
						if isinstance(SCREAMING_SNAKE_CASE_							,       (list, tuple)		):
								lowerCamelCase   : List[Any]								=model[0]
						lowerCamelCase   : Dict								=model.__name__
						if model not in in_table.values():
								missing.append(SCREAMING_SNAKE_CASE_		)

		if len(SCREAMING_SNAKE_CASE_		) > 0:
				lowerCamelCase   : Union[str, Any]								=''', '''.join(SCREAMING_SNAKE_CASE_		)
				raise ValueError(
				    '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
				    F"`utils/update_metadata.py`: {msg}. Please add them!"		)


if __name__ == "__main__":
	snake_case_							=			argparse.ArgumentParser()
	parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
	parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
	parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
	snake_case_							=			parser.parse_args()

	if args.check_only:
		check_pipeline_tags()
	else:
		update_metadata(args.token, args.commit_sha)








from pathlib import Path
from typing import List

from transformers import is_torch_available, is_vision_available
from transformers.testing_utils import get_tests_dir, is_tool_test
from transformers.tools.agent_types import AGENT_TYPE_MAPPING, AgentAudio, AgentImage, AgentText


if is_torch_available():
				import torch

if is_vision_available():
				from PIL import Image


snake_case_							=			['''text''', '''image''', '''audio''']


def 					A__	(       SCREAMING_SNAKE_CASE_		)   ->     List[str]:
		lowerCamelCase   : Union[str, Any]								=[]

		for input_type in input_types:
				if input_type == "text":
						inputs.append('''Text input'''		)
				elif input_type == "image":
						inputs.append(
						    Image.open(Path(get_tests_dir('''fixtures/tests_samples/COCO'''		)		) / '''000000039769.png'''		).resize((5_1_2, 5_1_2)		)		)
				elif input_type == "audio":
						inputs.append(torch.ones(3_0_0_0		)		)
				elif isinstance(SCREAMING_SNAKE_CASE_							,       SCREAMING_SNAKE_CASE_		):
						inputs.append(create_inputs(SCREAMING_SNAKE_CASE_		)		)
				else:
						raise ValueError(F"Invalid type requested: {input_type}"		)

		return inputs


def 					A__	(       SCREAMING_SNAKE_CASE_		)   ->     Optional[int]:
		lowerCamelCase   : str								=[]

		for output in outputs:
				if isinstance(SCREAMING_SNAKE_CASE_							,       (str, AgentText)		):
						output_types.append('''text'''		)
				elif isinstance(SCREAMING_SNAKE_CASE_							,       (Image.Image, AgentImage)		):
						output_types.append('''image'''		)
				elif isinstance(SCREAMING_SNAKE_CASE_							,       (torch.Tensor, AgentAudio)		):
						output_types.append('''audio'''		)
				else:
						raise ValueError(F"Invalid output: {output}"		)

		return output_types



@is_tool_test
class        snake_case_		:







		def 			__lowercase   (		self   )				->   Tuple:
				self.assertTrue(hasattr(self.tool			,     '''inputs'''   )   )
				self.assertTrue(hasattr(self.tool			,     '''outputs'''   )   )

				lowerCamelCase   : List[Any]								=self.tool.inputs
				for _input in inputs:
						if isinstance(_input			,     __lowercase   ):
								for __input in _input:
										self.assertTrue(__input in authorized_types   )
						else:
								self.assertTrue(_input in authorized_types   )

				lowerCamelCase   : List[Any]								=self.tool.outputs
				for _output in outputs:
						self.assertTrue(_output in authorized_types   )







		def 			__lowercase   (		self   )				->   Any:
				lowerCamelCase   : Dict								=create_inputs(self.tool.inputs   )
				lowerCamelCase   : Any								=self.tool(*__lowercase   )

				# There is a single output
				if len(self.tool.outputs   ) == 1:
						lowerCamelCase   : Union[str, Any]								=[outputs]

				self.assertListEqual(output_types(__lowercase   )			,     self.tool.outputs   )







		def 			__lowercase   (		self   )				->   Any:
				self.assertTrue(hasattr(self.tool			,     '''description'''   )   )
				self.assertTrue(hasattr(self.tool			,     '''default_checkpoint'''   )   )
				self.assertTrue(self.tool.description.startswith('''This is a tool that'''   )   )







		def 			__lowercase   (		self   )				->   Any:
				lowerCamelCase   : Dict								=create_inputs(self.tool.inputs   )
				lowerCamelCase   : Dict								=self.tool(*__lowercase   )

				if not isinstance(__lowercase			,     __lowercase   ):
						lowerCamelCase   : int								=[outputs]

				self.assertEqual(len(__lowercase   )			,     len(self.tool.outputs   )   )

				for output, output_type in zip(__lowercase			,     self.tool.outputs   ):
						lowerCamelCase   : Any								=AGENT_TYPE_MAPPING[output_type]
						self.assertTrue(isinstance(__lowercase			,     __lowercase   )   )







		def 			__lowercase   (		self   )				->   List[str]:
				lowerCamelCase   : Optional[Any]								=create_inputs(self.tool.inputs   )

				lowerCamelCase   : str								=[]

				for _input, input_type in zip(__lowercase			,     self.tool.inputs   ):
						if isinstance(__lowercase			,     __lowercase   ):
								_inputs.append([AGENT_TYPE_MAPPING[_input_type](_input   ) for _input_type in input_type]   )
						else:
								_inputs.append(AGENT_TYPE_MAPPING[input_type](_input   )   )

        # Should not raise an error
				lowerCamelCase   : Union[str, Any]								=self.tool(*__lowercase   )

				if not isinstance(__lowercase			,     __lowercase   ):
						lowerCamelCase   : int								=[outputs]

				self.assertEqual(len(__lowercase   )			,     len(self.tool.outputs   )   )

