



'''simple docstring'''






import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
UpperCamelCase__:     Optional[int]										=    "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
UpperCamelCase__:     List[Any]										=    direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
UpperCamelCase__:     Optional[Any]										=    re.compile(r"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
UpperCamelCase__:     Tuple										=    re.compile(r"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
UpperCamelCase__:     Dict										=    re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
UpperCamelCase__:     Optional[int]										=    [
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]


def 				snake_case_  (      _lowerCAmelCase						:		List[str]						)   ->     Any:
							UpperCAmelCase				: Any											=	re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''     ,   _lowerCAmelCase						)
							return [m.group(0						) for m in matches]


def 				snake_case_  (      )   ->     List[str]:
							UpperCAmelCase				: str											=	transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
							UpperCAmelCase				: List[str]											=	{
							    config.replace('''Config'''     ,   ''''''						): model_type for model_type, config in config_maping_names.items()
							}

							# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
							UpperCAmelCase				: Optional[Any]											=	collections.defaultdict(_lowerCAmelCase						)
							UpperCAmelCase				: Tuple											=	collections.defaultdict(_lowerCAmelCase						)
							UpperCAmelCase				: List[Any]											=	collections.defaultdict(_lowerCAmelCase						)

							# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
							for attr_name in dir(_lowerCAmelCase						):
														UpperCAmelCase				: Union[str, Any]											=	None
														if _re_tf_models.match(_lowerCAmelCase						) is not None:
																					UpperCAmelCase				: List[str]											=	tf_models
																					UpperCAmelCase				: Optional[Any]											=	_re_tf_models.match(_lowerCAmelCase						).groups()[0]
														elif _re_flax_models.match(_lowerCAmelCase						) is not None:
																					UpperCAmelCase				: str											=	flax_models
																					UpperCAmelCase				: Dict											=	_re_flax_models.match(_lowerCAmelCase						).groups()[0]
														elif _re_pt_models.match(_lowerCAmelCase						) is not None:
																					UpperCAmelCase				: Dict											=	pt_models
																					UpperCAmelCase				: List[str]											=	_re_pt_models.match(_lowerCAmelCase						).groups()[0]

														if lookup_dict is not None:
																					while len(_lowerCAmelCase						) > 0:
																												if attr_name in model_prefix_to_model_type:
																																			UpperCAmelCase				: Tuple											=	True
																																			break
																												# Try again after removing the last word in the name
																												UpperCAmelCase				: Dict											=	''''''.join(camel_case_split(_lowerCAmelCase						)[:-1]						)

							UpperCAmelCase				: Optional[int]											=	set(list(pt_models.keys()						) + list(tf_models.keys()						) + list(flax_models.keys()						)						)
							UpperCAmelCase				: Optional[int]											=	list(_lowerCAmelCase						)
							all_models.sort()

							UpperCAmelCase				: List[Any]											=	{'''model_type''': all_models}
							UpperCAmelCase				: Tuple											=	[pt_models[t] for t in all_models]
							UpperCAmelCase				: Any											=	[tf_models[t] for t in all_models]
							UpperCAmelCase				: Union[str, Any]											=	[flax_models[t] for t in all_models]

							# Now let's use the auto-mapping names to make sure
							UpperCAmelCase				: Union[str, Any]											=	{}
							for t in all_models:
														if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
																					UpperCAmelCase				: Any											=	'''AutoProcessor'''
														elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
																					UpperCAmelCase				: int											=	'''AutoTokenizer'''
														elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
																					UpperCAmelCase				: Optional[Any]											=	'''AutoFeatureExtractor'''
														else:
																					# Default to AutoTokenizer if a model has nothing, for backward compatibility.
																					UpperCAmelCase				: Union[str, Any]											=	'''AutoTokenizer'''

							UpperCAmelCase				: Optional[int]											=	[processors[t] for t in all_models]

							return pd.DataFrame(_lowerCAmelCase						)


def 				snake_case_  (      _lowerCAmelCase						:		Optional[Any]						)   ->     Optional[Any]:
							UpperCAmelCase				: Any											=	[
							    transformers_module.models.auto.modeling_auto,
							    transformers_module.models.auto.modeling_tf_auto,
							    transformers_module.models.auto.modeling_flax_auto,
							]
							for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
														UpperCAmelCase				: str											=	[model_mapping, f"""TF_{model_mapping}""", f"""FLAX_{model_mapping}"""]
														UpperCAmelCase				: Any											=	[auto_class, f"""TF_{auto_class}""", f"""Flax_{auto_class}"""]
														# Loop through all three frameworks
														for module, cls, mapping in zip(_lowerCAmelCase     ,   _lowerCAmelCase     ,   _lowerCAmelCase						):
																					# The type of pipeline may not exist in this framework
																					if not hasattr(_lowerCAmelCase     ,   _lowerCAmelCase						):
																												continue
																					# First extract all model_names
																					UpperCAmelCase				: int											=	[]
																					for name in getattr(_lowerCAmelCase     ,   _lowerCAmelCase						).values():
																												if isinstance(_lowerCAmelCase     ,   _lowerCAmelCase						):
																																			model_names.append(_lowerCAmelCase						)
																												else:
																																			model_names.extend(list(_lowerCAmelCase						)						)

            # Add pipeline tag and auto model class for those models
																					table.update({model_name: (pipeline_tag, cls) for model_name in model_names}						)

							return table


def 				snake_case_  (      _lowerCAmelCase						:		List[str]     ,   _lowerCAmelCase						:		Optional[Any]						)   ->     Any:
							UpperCAmelCase				: Union[str, Any]											=	get_frameworks_table()
							UpperCAmelCase				: Optional[int]											=	Dataset.from_pandas(_lowerCAmelCase						)

							UpperCAmelCase				: List[str]											=	hf_hub_download(
							    '''huggingface/transformers-metadata'''     ,   '''pipeline_tags.json'''     ,   repo_type='''dataset'''     ,   token=_lowerCAmelCase						)
							UpperCAmelCase				: List[Any]											=	Dataset.from_json(_lowerCAmelCase						)
							UpperCAmelCase				: int											=	{
							    tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
							    for i in range(len(_lowerCAmelCase						)						)
							}
							UpperCAmelCase				: str											=	update_pipeline_and_auto_class_table(_lowerCAmelCase						)

							# Sort the model classes to avoid some nondeterministic updates to create false update commits.
							UpperCAmelCase				: Any											=	sorted(table.keys()						)
							UpperCAmelCase				: int											=	pd.DataFrame(
							    {
							        '''model_class''': model_classes,
							        '''pipeline_tag''': [table[m][0] for m in model_classes],
							        '''auto_class''': [table[m][1] for m in model_classes],
							    }						)
							UpperCAmelCase				: Optional[Any]											=	Dataset.from_pandas(_lowerCAmelCase						)

							with tempfile.TemporaryDirectory() as tmp_dir:
														frameworks_dataset.to_json(os.path.join(_lowerCAmelCase     ,   '''frameworks.json'''						)						)
														tags_dataset.to_json(os.path.join(_lowerCAmelCase     ,   '''pipeline_tags.json'''						)						)

														if commit_sha is not None:
																					UpperCAmelCase				: Union[str, Any]											=	(
																					    f"""Update with commit {commit_sha}\n\nSee: """
																					    f"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
																					)
														else:
																					UpperCAmelCase				: List[str]											=	'''Update'''

														upload_folder(
														    repo_id='''huggingface/transformers-metadata'''     ,   folder_path=_lowerCAmelCase     ,   repo_type='''dataset'''     ,   token=_lowerCAmelCase     ,   commit_message=_lowerCAmelCase     ,   )


def 				snake_case_  (      )   ->     List[Any]:
							UpperCAmelCase				: Optional[int]											=	{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
							UpperCAmelCase				: Any											=	transformers_module.pipelines.SUPPORTED_TASKS
							UpperCAmelCase				: Any											=	[]
							for key in pipeline_tasks:
														if key not in in_table:
																					UpperCAmelCase				: Union[str, Any]											=	pipeline_tasks[key]['''pt''']
																					if isinstance(_lowerCAmelCase     ,   (list, tuple)						):
																												UpperCAmelCase				: Optional[Any]											=	model[0]
																					UpperCAmelCase				: str											=	model.__name__
																					if model not in in_table.values():
																												missing.append(_lowerCAmelCase						)

							if len(_lowerCAmelCase						) > 0:
														UpperCAmelCase				: str											=	''', '''.join(_lowerCAmelCase						)
														raise ValueError(
														    '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
														    f"""`utils/update_metadata.py`: {msg}. Please add them!"""						)


if __name__ == "__main__":
						UpperCamelCase__:     str										=    argparse.ArgumentParser()
						parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
						parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
						parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
						UpperCamelCase__:     int										=    parser.parse_args()

						if args.check_only:
												check_pipeline_tags()
						else:
												update_metadata(args.token, args.commit_sha)










'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__:     Dict										=    logging.get_logger(__name__)

UpperCamelCase__:     int										=    {
    # See all MEGATRON_BERT models at https://huggingface.co/models?filter=bert
}

class        SCREAMING_SNAKE_CASE(							A__    ):







	"""simple docstring"""






	lowerCamelCase__    				= """megatron-bert"""







	def __init__(		self    :  int	,    __snake_case    :  List[Any]=29056	,    __snake_case    :  Dict=1024	,    __snake_case    :  Tuple=24	,    __snake_case    :  Any=16	,    __snake_case    :  Tuple=4096	,    __snake_case    :  int="gelu"	,    __snake_case    :  Dict=0.1	,    __snake_case    :  str=0.1	,    __snake_case    :  Optional[Any]=512	,    __snake_case    :  Tuple=2	,    __snake_case    :  Tuple=0.02	,    __snake_case    :  int=1E-12	,    __snake_case    :  Union[str, Any]=0	,    __snake_case    :  int="absolute"	,    __snake_case    :  str=True	,    **__snake_case    :  Tuple	,    )    ->	Union[str, Any]:
								super().__init__(pad_token_id=__snake_case	,    **__snake_case	)

								UpperCAmelCase				: Any											=	vocab_size
								UpperCAmelCase				: Tuple											=	hidden_size
								UpperCAmelCase				: List[Any]											=	num_hidden_layers
								UpperCAmelCase				: str											=	num_attention_heads
								UpperCAmelCase				: Optional[int]											=	hidden_act
								UpperCAmelCase				: Optional[int]											=	intermediate_size
								UpperCAmelCase				: Dict											=	hidden_dropout_prob
								UpperCAmelCase				: List[Any]											=	attention_probs_dropout_prob
								UpperCAmelCase				: Dict											=	max_position_embeddings
								UpperCAmelCase				: Any											=	type_vocab_size
								UpperCAmelCase				: str											=	initializer_range
								UpperCAmelCase				: str											=	layer_norm_eps
								UpperCAmelCase				: Optional[Any]											=	position_embedding_type
								UpperCAmelCase				: str											=	use_cache





