



import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
_lowercase						  =		"src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
_lowercase						  =		direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
_lowercase						  =		re.compile(R"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
_lowercase						  =		re.compile(R"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
_lowercase						  =		re.compile(R"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
_lowercase						  =		[
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]



def 		lowerCAmelCase__    (					UpperCamelCase_				:       str	)->  Dict:
    A__		 =					re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)''' ,       UpperCamelCase_	)
    return [m.group(0	) for m in matches]



def 		lowerCAmelCase__    (					)->  int:
    A__		 =					transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
    A__		 =					{
        config.replace('''Config''' ,       ''''''	): model_type for model_type, config in config_maping_names.items()
    }

    # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
    A__		 =					collections.defaultdict(UpperCamelCase_	)
    A__		 =					collections.defaultdict(UpperCamelCase_	)
    A__		 =					collections.defaultdict(UpperCamelCase_	)

    # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
    for attr_name in dir(UpperCamelCase_	):
        A__		 =					None
        if _re_tf_models.match(UpperCamelCase_	) is not None:
            A__		 =					tf_models
            A__		 =					_re_tf_models.match(UpperCamelCase_	).groups()[0]
        elif _re_flax_models.match(UpperCamelCase_	) is not None:
            A__		 =					flax_models
            A__		 =					_re_flax_models.match(UpperCamelCase_	).groups()[0]
        elif _re_pt_models.match(UpperCamelCase_	) is not None:
            A__		 =					pt_models
            A__		 =					_re_pt_models.match(UpperCamelCase_	).groups()[0]

        if lookup_dict is not None:
            while len(UpperCamelCase_	) > 0:
                if attr_name in model_prefix_to_model_type:
                    A__		 =					True
                    break
                # Try again after removing the last word in the name
                A__		 =					''''''.join(camel_case_split(UpperCamelCase_	)[:-1]	)

    A__		 =					set(list(pt_models.keys()	) + list(tf_models.keys()	) + list(flax_models.keys()	)	)
    A__		 =					list(UpperCamelCase_	)
    all_models.sort()

    A__		 =					{'''model_type''': all_models}
    A__		 =					[pt_models[t] for t in all_models]
    A__		 =					[tf_models[t] for t in all_models]
    A__		 =					[flax_models[t] for t in all_models]

    # Now let's use the auto-mapping names to make sure
    A__		 =					{}
    for t in all_models:
        if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
            A__		 =					'''AutoProcessor'''
        elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
            A__		 =					'''AutoTokenizer'''
        elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
            A__		 =					'''AutoFeatureExtractor'''
        else:
            # Default to AutoTokenizer if a model has nothing, for backward compatibility.
            A__		 =					'''AutoTokenizer'''

    A__		 =					[processors[t] for t in all_models]

    return pd.DataFrame(UpperCamelCase_	)



def 		lowerCAmelCase__    (					UpperCamelCase_				:       Optional[int]	)->  str:
    A__		 =					[
        transformers_module.models.auto.modeling_auto,
        transformers_module.models.auto.modeling_tf_auto,
        transformers_module.models.auto.modeling_flax_auto,
    ]
    for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
        A__		 =					[model_mapping, f"TF_{model_mapping}", f"FLAX_{model_mapping}"]
        A__		 =					[auto_class, f"TF_{auto_class}", f"Flax_{auto_class}"]
        # Loop through all three frameworks
        for module, cls, mapping in zip(UpperCamelCase_ ,       UpperCamelCase_ ,       UpperCamelCase_	):
            # The type of pipeline may not exist in this framework
            if not hasattr(UpperCamelCase_ ,       UpperCamelCase_	):
                continue
            # First extract all model_names
            A__		 =					[]
            for name in getattr(UpperCamelCase_ ,       UpperCamelCase_	).values():
                if isinstance(UpperCamelCase_ ,       UpperCamelCase_	):
                    model_names.append(UpperCamelCase_	)
                else:
                    model_names.extend(list(UpperCamelCase_	)	)

            # Add pipeline tag and auto model class for those models
            table.update({model_name: (pipeline_tag, cls) for model_name in model_names}	)

    return table



def 		lowerCAmelCase__    (					UpperCamelCase_				:       List[Any] ,       UpperCamelCase_				:       List[Any]	)->  Optional[Any]:
    A__		 =					get_frameworks_table()
    A__		 =					Dataset.from_pandas(UpperCamelCase_	)

    A__		 =					hf_hub_download(
        '''huggingface/transformers-metadata''' ,       '''pipeline_tags.json''' ,       repo_type='''dataset''' ,       token=UpperCamelCase_	)
    A__		 =					Dataset.from_json(UpperCamelCase_	)
    A__		 =					{
        tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
        for i in range(len(UpperCamelCase_	)	)
    }
    A__		 =					update_pipeline_and_auto_class_table(UpperCamelCase_	)

    # Sort the model classes to avoid some nondeterministic updates to create false update commits.
    A__		 =					sorted(table.keys()	)
    A__		 =					pd.DataFrame(
        {
            '''model_class''': model_classes,
            '''pipeline_tag''': [table[m][0] for m in model_classes],
            '''auto_class''': [table[m][1] for m in model_classes],
        }	)
    A__		 =					Dataset.from_pandas(UpperCamelCase_	)

    with tempfile.TemporaryDirectory() as tmp_dir:
        frameworks_dataset.to_json(os.path.join(UpperCamelCase_ ,       '''frameworks.json'''	)	)
        tags_dataset.to_json(os.path.join(UpperCamelCase_ ,       '''pipeline_tags.json'''	)	)

        if commit_sha is not None:
            A__		 =					(
                f"Update with commit {commit_sha}\n\nSee: "
                f"https://github.com/huggingface/transformers/commit/{commit_sha}"
            )
        else:
            A__		 =					'''Update'''

        upload_folder(
            repo_id='''huggingface/transformers-metadata''' ,       folder_path=UpperCamelCase_ ,       repo_type='''dataset''' ,       token=UpperCamelCase_ ,       commit_message=UpperCamelCase_ ,       )



def 		lowerCAmelCase__    (					)->  List[Any]:
    A__		 =					{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
    A__		 =					transformers_module.pipelines.SUPPORTED_TASKS
    A__		 =					[]
    for key in pipeline_tasks:
        if key not in in_table:
            A__		 =					pipeline_tasks[key]['''pt''']
            if isinstance(UpperCamelCase_ ,       (list, tuple)	):
                A__		 =					model[0]
            A__		 =					model.__name__
            if model not in in_table.values():
                missing.append(UpperCamelCase_	)

    if len(UpperCamelCase_	) > 0:
        A__		 =					''', '''.join(UpperCamelCase_	)
        raise ValueError(
            '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
            f"`utils/update_metadata.py`: {msg}. Please add them!"	)


if __name__ == "__main__":
   _lowercase						  =		argparse.ArgumentParser()
   parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
   parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
   parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
   _lowercase						  =		parser.parse_args()

   if args.check_only:
      check_pipeline_tags()
   else:
      update_metadata(args.token, args.commit_sha)









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


_lowercase						  =		logging.get_logger(__name__)

_lowercase						  =		{
    "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/config.json",
    "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/config.json",
    "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/config.json",
    "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/config.json",
    "bert-base-multilingual-uncased": "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/config.json",
    "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/config.json",
    "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/config.json",
    "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/config.json",
    "bert-large-uncased-whole-word-masking": (
        "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/config.json"
    ),
    "bert-large-cased-whole-word-masking": (
        "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/config.json"
    ),
    "bert-large-uncased-whole-word-masking-finetuned-squad": (
        "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/config.json"
    ),
    "bert-large-cased-whole-word-masking-finetuned-squad": (
        "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/config.json"
    ),
    "bert-base-cased-finetuned-mrpc": "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/config.json",
    "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/config.json",
    "bert-base-german-dbmdz-uncased": "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/config.json",
    "cl-tohoku/bert-base-japanese": "https://huggingface.co/cl-tohoku/bert-base-japanese/resolve/main/config.json",
    "cl-tohoku/bert-base-japanese-whole-word-masking": (
        "https://huggingface.co/cl-tohoku/bert-base-japanese-whole-word-masking/resolve/main/config.json"
    ),
    "cl-tohoku/bert-base-japanese-char": (
        "https://huggingface.co/cl-tohoku/bert-base-japanese-char/resolve/main/config.json"
    ),
    "cl-tohoku/bert-base-japanese-char-whole-word-masking": (
        "https://huggingface.co/cl-tohoku/bert-base-japanese-char-whole-word-masking/resolve/main/config.json"
    ),
    "TurkuNLP/bert-base-finnish-cased-v1": (
        "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/config.json"
    ),
    "TurkuNLP/bert-base-finnish-uncased-v1": (
        "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/config.json"
    ),
    "wietsedv/bert-base-dutch-cased": "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/config.json",
    # See all BERT models at https://huggingface.co/models?filter=bert
}



class     _UpperCAmelCase			(       A__     ):
     UpperCamelCase__		    =				'''bert'''




     def __init__(				self ,						a__=3_0_5_2_2 ,						a__=7_6_8 ,						a__=1_2 ,						a__=1_2 ,						a__=3_0_7_2 ,						a__="gelu" ,						a__=0.1 ,						a__=0.1 ,						a__=5_1_2 ,						a__=2 ,						a__=0.0_2 ,						a__=1e-12 ,						a__=0 ,						a__="absolute" ,						a__=True ,						a__=None ,						**a__ ,						):
         super().__init__(pad_token_id=a__ ,						**a__)

         A__		 =					vocab_size
         A__		 =					hidden_size
         A__		 =					num_hidden_layers
         A__		 =					num_attention_heads
         A__		 =					hidden_act
         A__		 =					intermediate_size
         A__		 =					hidden_dropout_prob
         A__		 =					attention_probs_dropout_prob
         A__		 =					max_position_embeddings
         A__		 =					type_vocab_size
         A__		 =					initializer_range
         A__		 =					layer_norm_eps
         A__		 =					position_embedding_type
         A__		 =					use_cache
         A__		 =					classifier_dropout







class     _UpperCAmelCase			(       A__     ):




     @property
     def 		snake_case_		(				self):
         if self.task == "multiple-choice":
             A__		 =					{0: '''batch''', 1: '''choice''', 2: '''sequence'''}
         else:
             A__		 =					{0: '''batch''', 1: '''sequence'''}
         return OrderedDict(
             [
                 ('''input_ids''', dynamic_axis),
                 ('''attention_mask''', dynamic_axis),
                 ('''token_type_ids''', dynamic_axis),
             ])
