




'''simple docstring'''


import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
_snake_case   :   str     							=	'''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
_snake_case   :   Dict     							=	direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
_snake_case   :   Dict     							=	re.compile(R'TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
_snake_case   :   int     							=	re.compile(R'Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
_snake_case   :   Dict     							=	re.compile(R'(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
_snake_case   :   Optional[Any]     							=	[
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]
def 	snake_case_					(UpperCamelCase :						int   ):





							'''simple docstring'''


							_a			  =       re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''				,		UpperCamelCase   )
							return [m.group(0   ) for m in matches]
def 	snake_case_					():





							'''simple docstring'''


							_a			  =       transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
							_a			  =       {
							    config.replace('''Config'''				,		''''''   ): model_type for model_type, config in config_maping_names.items()
							}

							# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
							_a			  =       collections.defaultdict(UpperCamelCase   )
							_a			  =       collections.defaultdict(UpperCamelCase   )
							_a			  =       collections.defaultdict(UpperCamelCase   )

							# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
							for attr_name in dir(UpperCamelCase   ):
														_a			  =       None
														if _re_tf_models.match(UpperCamelCase   ) is not None:
																					_a			  =       tf_models
																					_a			  =       _re_tf_models.match(UpperCamelCase   ).groups()[0]
														elif _re_flax_models.match(UpperCamelCase   ) is not None:
																					_a			  =       flax_models
																					_a			  =       _re_flax_models.match(UpperCamelCase   ).groups()[0]
														elif _re_pt_models.match(UpperCamelCase   ) is not None:
																					_a			  =       pt_models
																					_a			  =       _re_pt_models.match(UpperCamelCase   ).groups()[0]

														if lookup_dict is not None:
																					while len(UpperCamelCase   ) > 0:
																												if attr_name in model_prefix_to_model_type:
																																			_a			  =       True
																																			break
																												# Try again after removing the last word in the name
																												_a			  =       ''''''.join(camel_case_split(UpperCamelCase   )[:-1]   )

							_a			  =       set(list(pt_models.keys()   ) + list(tf_models.keys()   ) + list(flax_models.keys()   )   )
							_a			  =       list(UpperCamelCase   )
							all_models.sort()

							_a			  =       {'''model_type''': all_models}
							_a			  =       [pt_models[t] for t in all_models]
							_a			  =       [tf_models[t] for t in all_models]
							_a			  =       [flax_models[t] for t in all_models]

							# Now let's use the auto-mapping names to make sure
							_a			  =       {}
							for t in all_models:
														if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
																					_a			  =       '''AutoProcessor'''
														elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
																					_a			  =       '''AutoTokenizer'''
														elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
																					_a			  =       '''AutoFeatureExtractor'''
														else:
																					# Default to AutoTokenizer if a model has nothing, for backward compatibility.
																					_a			  =       '''AutoTokenizer'''

							_a			  =       [processors[t] for t in all_models]

							return pd.DataFrame(UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						Tuple   ):





							'''simple docstring'''


							_a			  =       [
							    transformers_module.models.auto.modeling_auto,
							    transformers_module.models.auto.modeling_tf_auto,
							    transformers_module.models.auto.modeling_flax_auto,
							]
							for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
														_a			  =       [model_mapping, f'TF_{model_mapping}', f'FLAX_{model_mapping}']
														_a			  =       [auto_class, f'TF_{auto_class}', f'Flax_{auto_class}']
														# Loop through all three frameworks
														for module, cls, mapping in zip(UpperCamelCase				,		UpperCamelCase				,		UpperCamelCase   ):
																					# The type of pipeline may not exist in this framework
																					if not hasattr(UpperCamelCase				,		UpperCamelCase   ):
																												continue
																					# First extract all model_names
																					_a			  =       []
																					for name in getattr(UpperCamelCase				,		UpperCamelCase   ).values():
																												if isinstance(UpperCamelCase				,		UpperCamelCase   ):
																																			model_names.append(UpperCamelCase   )
																												else:
																																			model_names.extend(list(UpperCamelCase   )   )

            # Add pipeline tag and auto model class for those models
																					table.update({model_name: (pipeline_tag, cls) for model_name in model_names}   )

							return table
def 	snake_case_					(UpperCamelCase :						Any				,		UpperCamelCase :						Any   ):





							'''simple docstring'''


							_a			  =       get_frameworks_table()
							_a			  =       Dataset.from_pandas(UpperCamelCase   )

							_a			  =       hf_hub_download(
							    '''huggingface/transformers-metadata'''				,		'''pipeline_tags.json'''				,		repo_type='''dataset'''				,		token=UpperCamelCase   )
							_a			  =       Dataset.from_json(UpperCamelCase   )
							_a			  =       {
							    tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
							    for i in range(len(UpperCamelCase   )   )
							}
							_a			  =       update_pipeline_and_auto_class_table(UpperCamelCase   )

							# Sort the model classes to avoid some nondeterministic updates to create false update commits.
							_a			  =       sorted(table.keys()   )
							_a			  =       pd.DataFrame(
							    {
							        '''model_class''': model_classes,
							        '''pipeline_tag''': [table[m][0] for m in model_classes],
							        '''auto_class''': [table[m][1] for m in model_classes],
							    }   )
							_a			  =       Dataset.from_pandas(UpperCamelCase   )

							with tempfile.TemporaryDirectory() as tmp_dir:
														frameworks_dataset.to_json(os.path.join(UpperCamelCase				,		'''frameworks.json'''   )   )
														tags_dataset.to_json(os.path.join(UpperCamelCase				,		'''pipeline_tags.json'''   )   )

														if commit_sha is not None:
																					_a			  =       (
																					    f'Update with commit {commit_sha}\n\nSee: '
																					    f'https://github.com/huggingface/transformers/commit/{commit_sha}'
																					)
														else:
																					_a			  =       '''Update'''

														upload_folder(
														    repo_id='''huggingface/transformers-metadata'''				,		folder_path=UpperCamelCase				,		repo_type='''dataset'''				,		token=UpperCamelCase				,		commit_message=UpperCamelCase				,		)
def 	snake_case_					():





							'''simple docstring'''


							_a			  =       {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
							_a			  =       transformers_module.pipelines.SUPPORTED_TASKS
							_a			  =       []
							for key in pipeline_tasks:
														if key not in in_table:
																					_a			  =       pipeline_tasks[key]['''pt''']
																					if isinstance(UpperCamelCase				,		(list, tuple)   ):
																												_a			  =       model[0]
																					_a			  =       model.__name__
																					if model not in in_table.values():
																												missing.append(UpperCamelCase   )

							if len(UpperCamelCase   ) > 0:
														_a			  =       ''', '''.join(UpperCamelCase   )
														raise ValueError(
														    '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
														    f'`utils/update_metadata.py`: {msg}. Please add them!'   )


if __name__ == "__main__":
	_snake_case   :   Dict     							=	argparse.ArgumentParser()
	parser.add_argument('--token', type=str, help='The token to use to push to the transformers-metadata dataset.')
	parser.add_argument('--commit_sha', type=str, help='The sha of the commit going with this update.')
	parser.add_argument('--check-only', action='store_true', help='Activate to just check all pipelines are present.')
	_snake_case   :   int     							=	parser.parse_args()

	if args.check_only:
		check_pipeline_tags()
	else:
		update_metadata(args.token, args.commit_sha)







'''simple docstring'''


import os

import pytest

from transformers.dynamic_module_utils import get_imports


_snake_case   :   List[str]     							=	'\nimport os\n'

_snake_case   :   Dict     							=	'\ndef foo():\n    import os\n    return False\n'

_snake_case   :   List[Any]     							=	'\ndef foo():\n    def bar():\n        if True:\n            import os\n        return False\n    return bar()\n'

_snake_case   :   Dict     							=	'\nimport os\n\ntry:\n    import bar\nexcept ImportError:\n    raise ValueError()\n'

_snake_case   :   Optional[Any]     							=	'\nimport os\n\ndef foo():\n    try:\n        import bar\n    except ImportError:\n        raise ValueError()\n'

_snake_case   :   Optional[Any]     							=	'\nimport os\n\ntry:\n    import bar\nexcept (ImportError, AttributeError):\n    raise ValueError()\n'

_snake_case   :   Any     							=	'\nimport os\n\ntry:\n    import bar\nexcept ImportError as e:\n    raise ValueError()\n'

_snake_case   :   List[str]     							=	'\nimport os\n\ntry:\n    import bar\nexcept:\n    raise ValueError()\n'

_snake_case   :   Dict     							=	'\nimport os\n\ntry:\n    import bar\n    import baz\nexcept ImportError:\n    raise ValueError()\n'

_snake_case   :   Any     							=	'\nimport os\n\ntry:\n    import bar\n    import baz\nexcept ImportError:\n    x = 1\n    raise ValueError()\n'

_snake_case   :   Union[str, Any]     							=	[
    TOP_LEVEL_IMPORT,
    IMPORT_IN_FUNCTION,
    DEEPLY_NESTED_IMPORT,
    TOP_LEVEL_TRY_IMPORT,
    GENERIC_EXCEPT_IMPORT,
    MULTILINE_TRY_IMPORT,
    MULTILINE_BOTH_IMPORT,
    MULTIPLE_EXCEPTS_IMPORT,
    EXCEPT_AS_IMPORT,
    TRY_IMPORT_IN_FUNCTION,
]
@pytest.mark.parametrize('''case'''				,		UpperCamelCase   )
def 	snake_case_					(UpperCamelCase :						str				,		UpperCamelCase :						List[Any]   ):





							'''simple docstring'''


							_a			  =       os.path.join(UpperCamelCase				,		'''test_file.py'''   )
							with open(UpperCamelCase				,		'''w'''   ) as _tmp_file:
														_tmp_file.write(UpperCamelCase   )

							_a			  =       get_imports(UpperCamelCase   )
							assert parsed_imports == ["os"]

