





import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
UpperCAmelCase_							=				'src/transformers'


# This is to make sure the transformers module imported is the one in the repo.
UpperCAmelCase_							=				direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
UpperCAmelCase_							=				re.compile(r"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
UpperCAmelCase_							=				re.compile(r"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
UpperCAmelCase_							=				re.compile(r"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
UpperCAmelCase_							=				[
    ('pretraining', 'MODEL_FOR_PRETRAINING_MAPPING_NAMES', 'AutoModelForPreTraining'),
    ('feature-extraction', 'MODEL_MAPPING_NAMES', 'AutoModel'),
    ('audio-classification', 'MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForAudioClassification'),
    ('text-generation', 'MODEL_FOR_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForCausalLM'),
    ('automatic-speech-recognition', 'MODEL_FOR_CTC_MAPPING_NAMES', 'AutoModelForCTC'),
    ('image-classification', 'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForImageClassification'),
    ('image-segmentation', 'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES', 'AutoModelForImageSegmentation'),
    ('fill-mask', 'MODEL_FOR_MASKED_LM_MAPPING_NAMES', 'AutoModelForMaskedLM'),
    ('object-detection', 'MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES', 'AutoModelForObjectDetection'),
    (
        'zero-shot-object-detection',
        'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES',
        'AutoModelForZeroShotObjectDetection',
    ),
    ('question-answering', 'MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES', 'AutoModelForQuestionAnswering'),
    ('text2text-generation', 'MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES', 'AutoModelForSeq2SeqLM'),
    ('text-classification', 'MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForSequenceClassification'),
    ('automatic-speech-recognition', 'MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES', 'AutoModelForSpeechSeq2Seq'),
    (
        'table-question-answering',
        'MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForTableQuestionAnswering',
    ),
    ('token-classification', 'MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForTokenClassification'),
    ('multiple-choice', 'MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES', 'AutoModelForMultipleChoice'),
    (
        'next-sentence-prediction',
        'MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES',
        'AutoModelForNextSentencePrediction',
    ),
    (
        'audio-frame-classification',
        'MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForAudioFrameClassification',
    ),
    ('audio-xvector', 'MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES', 'AutoModelForAudioXVector'),
    (
        'document-question-answering',
        'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForDocumentQuestionAnswering',
    ),
    (
        'visual-question-answering',
        'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES',
        'AutoModelForVisualQuestionAnswering',
    ),
    ('image-to-text', 'MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES', 'AutoModelForVision2Seq'),
    (
        'zero-shot-image-classification',
        'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES',
        'AutoModelForZeroShotImageClassification',
    ),
    ('depth-estimation', 'MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES', 'AutoModelForDepthEstimation'),
    ('video-classification', 'MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES', 'AutoModelForVideoClassification'),
    ('mask-generation', 'MODEL_FOR_MASK_GENERATION_MAPPING_NAMES', 'AutoModelForMaskGeneration'),
]





def  __magic_name__  (				lowercase				)				-> Dict:

							"""simple docstring"""



							lowercase_	:   Optional[Any]  				=     re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""				,       _A				)
							return [m.group(0				) for m in matches]





def  __magic_name__  (				)				-> Optional[Any]:

							"""simple docstring"""



							lowercase_	:   Union[str, Any]  				=     transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
							lowercase_	:   Union[str, Any]  				=     {
							    config.replace("""Config"""				,       """"""				): model_type for model_type, config in config_maping_names.items()
							}

							# Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
							lowercase_	:   Union[str, Any]  				=     collections.defaultdict(_A				)
							lowercase_	:   Optional[Any]  				=     collections.defaultdict(_A				)
							lowercase_	:   int  				=     collections.defaultdict(_A				)

							# Let's lookup through all transformers object (once) and find if models are supported by a given backend.
							for attr_name in dir(_A				):
														lowercase_	:   Optional[Any]  				=     None
														if _re_tf_models.match(_A				) is not None:
																					lowercase_	:   List[str]  				=     tf_models
																					lowercase_	:   str  				=     _re_tf_models.match(_A				).groups()[0]
														elif _re_flax_models.match(_A				) is not None:
																					lowercase_	:   Optional[int]  				=     flax_models
																					lowercase_	:   List[Any]  				=     _re_flax_models.match(_A				).groups()[0]
														elif _re_pt_models.match(_A				) is not None:
																					lowercase_	:   int  				=     pt_models
																					lowercase_	:   List[str]  				=     _re_pt_models.match(_A				).groups()[0]

														if lookup_dict is not None:
																					while len(_A				) > 0:
																												if attr_name in model_prefix_to_model_type:
																																			lowercase_	:   Optional[Any]  				=     True
																																			break
																												# Try again after removing the last word in the name
																												lowercase_	:   List[str]  				=     ''.join(camel_case_split(_A				)[:-1]				)

							lowercase_	:   Any  				=     set(list(pt_models.keys()				) + list(tf_models.keys()				) + list(flax_models.keys()				)				)
							lowercase_	:   int  				=     list(_A				)
							all_models.sort()

							lowercase_	:   Optional[Any]  				=     {'model_type': all_models}
							lowercase_	:   List[str]  				=     [pt_models[t] for t in all_models]
							lowercase_	:   str  				=     [tf_models[t] for t in all_models]
							lowercase_	:   int  				=     [flax_models[t] for t in all_models]

							# Now let's use the auto-mapping names to make sure
							lowercase_	:   Any  				=     {}
							for t in all_models:
														if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
																					lowercase_	:   Optional[int]  				=     'AutoProcessor'
														elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
																					lowercase_	:   Union[str, Any]  				=     'AutoTokenizer'
														elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
																					lowercase_	:   Tuple  				=     'AutoFeatureExtractor'
														else:
																					# Default to AutoTokenizer if a model has nothing, for backward compatibility.
																					lowercase_	:   str  				=     'AutoTokenizer'

							lowercase_	:   str  				=     [processors[t] for t in all_models]

							return pd.DataFrame(_A				)





def  __magic_name__  (				lowercase				)				-> Dict:

							"""simple docstring"""



							lowercase_	:   List[Any]  				=     [
							    transformers_module.models.auto.modeling_auto,
							    transformers_module.models.auto.modeling_tf_auto,
							    transformers_module.models.auto.modeling_flax_auto,
							]
							for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
														lowercase_	:   Optional[Any]  				=     [model_mapping, f"""TF_{model_mapping}""", f"""FLAX_{model_mapping}"""]
														lowercase_	:   Optional[int]  				=     [auto_class, f"""TF_{auto_class}""", f"""Flax_{auto_class}"""]
														# Loop through all three frameworks
														for module, cls, mapping in zip(_A				,       _A				,       _A				):
																					# The type of pipeline may not exist in this framework
																					if not hasattr(_A				,       _A				):
																												continue
																					# First extract all model_names
																					lowercase_	:   List[str]  				=     []
																					for name in getattr(_A				,       _A				).values():
																												if isinstance(_A				,       _A				):
																																			model_names.append(_A				)
																												else:
																																			model_names.extend(list(_A				)				)

																																			  # Add pipeline tag and auto model class for those models
																					table.update({model_name: (pipeline_tag, cls) for model_name in model_names}				)

							return table





def  __magic_name__  (				lowercase				,       lowercase				)				-> str:

							"""simple docstring"""



							lowercase_	:   Optional[int]  				=     get_frameworks_table()
							lowercase_	:   List[str]  				=     Dataset.from_pandas(_A				)

							lowercase_	:   Optional[Any]  				=     hf_hub_download(
							    """huggingface/transformers-metadata"""				,       """pipeline_tags.json"""				,       repo_type="""dataset"""				,       token=_A				)
							lowercase_	:   List[Any]  				=     Dataset.from_json(_A				)
							lowercase_	:   Optional[Any]  				=     {
							    tags_dataset[i]['model_class']: (tags_dataset[i]['pipeline_tag'], tags_dataset[i]['auto_class'])
							    for i in range(len(_A				)				)
							}
							lowercase_	:   Dict  				=     update_pipeline_and_auto_class_table(_A				)

							# Sort the model classes to avoid some nondeterministic updates to create false update commits.
							lowercase_	:   str  				=     sorted(table.keys()				)
							lowercase_	:   List[Any]  				=     pd.DataFrame(
							    {
							        """model_class""": model_classes,
							        """pipeline_tag""": [table[m][0] for m in model_classes],
							        """auto_class""": [table[m][1] for m in model_classes],
							    }				)
							lowercase_	:   Optional[Any]  				=     Dataset.from_pandas(_A				)

							with tempfile.TemporaryDirectory() as tmp_dir:
														frameworks_dataset.to_json(os.path.join(_A				,       """frameworks.json"""				)				)
														tags_dataset.to_json(os.path.join(_A				,       """pipeline_tags.json"""				)				)

														if commit_sha is not None:
																					lowercase_	:   int  				=     (
																					    f"""Update with commit {commit_sha}\n\nSee: """
																					    f"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
																					)
														else:
																					lowercase_	:   Optional[int]  				=     'Update'

														upload_folder(
														    repo_id="""huggingface/transformers-metadata"""				,       folder_path=_A				,       repo_type="""dataset"""				,       token=_A				,       commit_message=_A				,       )





def  __magic_name__  (				)				-> List[str]:

							"""simple docstring"""



							lowercase_	:   int  				=     {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
							lowercase_	:   List[Any]  				=     transformers_module.pipelines.SUPPORTED_TASKS
							lowercase_	:   str  				=     []
							for key in pipeline_tasks:
														if key not in in_table:
																					lowercase_	:   Dict  				=     pipeline_tasks[key]['pt']
																					if isinstance(_A				,       (list, tuple)				):
																												lowercase_	:   List[Any]  				=     model[0]
																					lowercase_	:   Union[str, Any]  				=     model.__name__
																					if model not in in_table.values():
																												missing.append(_A				)

							if len(_A				) > 0:
														lowercase_	:   List[str]  				=     ', '.join(_A				)
														raise ValueError(
														    """The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside """
														    f"""`utils/update_metadata.py`: {msg}. Please add them!"""				)


if __name__ == "__main__":
							UpperCAmelCase_							=				argparse.ArgumentParser()
							parser.add_argument("""--token""", type=str, help="""The token to use to push to the transformers-metadata dataset.""")
							parser.add_argument("""--commit_sha""", type=str, help="""The sha of the commit going with this update.""")
							parser.add_argument("""--check-only""", action="""store_true""", help="""Activate to just check all pipelines are present.""")
							UpperCAmelCase_							=				parser.parse_args()

							if args.check_only:
														check_pipeline_tags()
							else:
														update_metadata(args.token, args.commit_sha)




'''simple docstring'''






import json
import os
from pathlib import Path
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple, Union

import sentencepiece

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase:   Tuple					=		logging.get_logger(__name__)

lowerCAmelCase:   Dict					=		'▁'

lowerCAmelCase:   Dict					=		{
    'vocab_file': 'vocab.json',
    'spm_file': 'sentencepiece.bpe.model',
}

lowerCAmelCase:   int					=		{
    'vocab_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/vocab.json'
        ),
    },
    'spm_file': {
        'facebook/s2t-small-librispeech-asr': (
            'https://huggingface.co/facebook/s2t-small-librispeech-asr/resolve/main/sentencepiece.bpe.model'
        )
    },
}

lowerCAmelCase:   Any					=		{
    'facebook/s2t-small-librispeech-asr': 1_0_2_4,
}

lowerCAmelCase:   Optional[int]					=		['pt', 'fr', 'ru', 'nl', 'ro', 'it', 'es', 'de']

lowerCAmelCase:   List[Any]					=		{'mustc': MUSTC_LANGS}

class 				a__(   lowerCamelCase__      ):
    lowercase__					=       VOCAB_FILES_NAMES
    lowercase__					=       PRETRAINED_VOCAB_FILES_MAP
    lowercase__					=       MAX_MODEL_INPUT_SIZES
    lowercase__					=       ["""input_ids""", """attention_mask"""]

    lowercase__					=       []







    def __init__( self					:    int		,     __snake_case					:    int		,     __snake_case					:    Optional[Any]		,     __snake_case					:    Optional[Any]="<s>"		,     __snake_case					:    Optional[int]="</s>"		,     __snake_case					:    Union[str, Any]="<pad>"		,     __snake_case					:    str="<unk>"		,     __snake_case					:    Dict=False		,     __snake_case					:    int=False		,     __snake_case					:    str=None		,     __snake_case					:    Optional[int]=None		,     __snake_case					:    Optional[Dict[str, Any]] = None		,     **__snake_case					:    Union[str, Any]		,     ):
      a		:   Union[str, Any]	  =	{} if sp_model_kwargs is None else sp_model_kwargs

      super().__init__(
          bos_token=__snake_case		,     eos_token=__snake_case		,     unk_token=__snake_case		,     pad_token=__snake_case		,     do_upper_case=__snake_case		,     do_lower_case=__snake_case		,     tgt_lang=__snake_case		,     lang_codes=__snake_case		,     sp_model_kwargs=self.sp_model_kwargs		,     **__snake_case		,     )
      a		:   Tuple	  =	do_upper_case
      a		:   Optional[Any]	  =	do_lower_case

      a		:   List[str]	  =	load_json(__snake_case      )
      a		:   Dict	  =	{v: k for k, v in self.encoder.items()}
      a		:   int	  =	spm_file
      a		:   Tuple	  =	load_spm(__snake_case		,     self.sp_model_kwargs      )

      if lang_codes is not None:
        a		:   Any	  =	lang_codes
        a		:   str	  =	LANGUAGES[lang_codes]
        a		:   Tuple	  =	[F"""<lang:{lang}>""" for lang in self.langs]
        a		:   str	  =	{lang: self.sp_model.PieceToId(F"""<lang:{lang}>"""      ) for lang in self.langs}

        a		:   Optional[Any]	  =	self.lang_tokens
        a		:   Union[str, Any]	  =	tgt_lang if tgt_lang is not None else self.langs[0]

        self.set_tgt_lang_special_tokens(self._tgt_lang      )
      else:
        a		:   List[str]	  =	{}







    @property
    def      lowercase_			( self					:    Optional[Any]      ):
      return len(self.encoder      )







    @property
    def      lowercase_			( self					:    int      ):
      return self._tgt_lang







    @tgt_lang.setter
    def      lowercase_			( self					:    int		,     __snake_case					:    Optional[int]      ):
      a		:   Union[str, Any]	  =	new_tgt_lang
      self.set_tgt_lang_special_tokens(__snake_case      )







    def      lowercase_			( self					:    str		,     __snake_case					:    str      ):
      a		:   int	  =	self.lang_code_to_id[tgt_lang]
      a		:   int	  =	[lang_code_id]







    def      lowercase_			( self					:    Optional[int]		,     __snake_case					:    str      ):
      return self.sp_model.encode(__snake_case		,     out_type=__snake_case      )







    def      lowercase_			( self					:    List[str]		,     __snake_case					:    List[Any]      ):
      return self.encoder.get(__snake_case		,     self.encoder[self.unk_token]      )







    def      lowercase_			( self					:    List[Any]		,     __snake_case					:    int      ):
      return self.decoder.get(__snake_case		,     self.unk_token      )







    def      lowercase_			( self					:    Union[str, Any]		,     __snake_case					:    List[str]      ):
      a		:   List[Any]	  =	[]
      a		:   List[Any]	  =	''
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          a		:   Union[str, Any]	  =	self.sp_model.decode(__snake_case      )
          out_string += (decoded.upper() if self.do_upper_case else decoded) + token + " "
          a		:   Optional[int]	  =	[]
        else:
          current_sub_tokens.append(__snake_case      )
      a		:   Tuple	  =	self.sp_model.decode(__snake_case      )
      out_string += decoded.upper() if self.do_upper_case else decoded
      return out_string.strip()







    def      lowercase_			( self					:    int		,     __snake_case					:    List[Any]		,     __snake_case					:    List[str]=None      ):
      if token_ids_a is None:
        return self.prefix_tokens + token_ids_a + [self.eos_token_id]
      # We don't expect to process pairs, but leave the pair logic for API consistency
      return self.prefix_tokens + token_ids_a + token_ids_a + [self.eos_token_id]







    def      lowercase_			( self					:    Optional[int]		,     __snake_case					:    List[int]		,     __snake_case					:    Optional[List[int]] = None		,     __snake_case					:    bool = False      ):

      if already_has_special_tokens:
        return super().get_special_tokens_mask(
            token_ids_a=__snake_case		,     token_ids_a=__snake_case		,     already_has_special_tokens=__snake_case      )

      a		:   Optional[int]	  =	[1] * len(self.prefix_tokens      )
      a		:   Any	  =	[1]
      if token_ids_a is None:
        return prefix_ones + ([0] * len(__snake_case      )) + suffix_ones
      return prefix_ones + ([0] * len(__snake_case      )) + ([0] * len(__snake_case      )) + suffix_ones







    def      lowercase_			( self					:    Union[str, Any]      ):
      a		:   Union[str, Any]	  =	self.encoder.copy()
      vocab.update(self.added_tokens_encoder      )
      return vocab







    def __getstate__( self					:    Any      ):
      a		:   List[str]	  =	self.__dict__.copy()
      a		:   Union[str, Any]	  =	None
      return state







    def __setstate__( self					:    str		,     __snake_case					:    Dict      ):
      a		:   Dict	  =	d

      # for backward compatibility
      if not hasattr(self		,     'sp_model_kwargs'      ):
        a		:   int	  =	{}

      a		:   Any	  =	load_spm(self.spm_file		,     self.sp_model_kwargs      )







    def      lowercase_			( self					:    str		,     __snake_case					:    str		,     __snake_case					:    Optional[str] = None      ):
      a		:   Union[str, Any]	  =	Path(__snake_case      )
      assert save_dir.is_dir(), F"""{save_directory} should be a directory"""
      a		:   Any	  =	save_dir / (
          (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['vocab_file']
      )
      a		:   List[Any]	  =	save_dir / (
          (filename_prefix + '-' if filename_prefix else '') + self.vocab_files_names['spm_file']
      )

      save_json(self.encoder		,     __snake_case      )

      if os.path.abspath(self.spm_file      ) != os.path.abspath(__snake_case      ) and os.path.isfile(self.spm_file      ):
        copyfile(self.spm_file		,     __snake_case      )
      elif not os.path.isfile(self.spm_file      ):
        with open(__snake_case		,     'wb'      ) as fi:
          a		:   Tuple	  =	self.sp_model.serialized_model_proto()
          fi.write(__snake_case      )

      return (str(__snake_case      ), str(__snake_case      ))
def  lowerCamelCase__			(		_A					,     _A							):
  a		:   List[Any]	  =	sentencepiece.SentencePieceProcessor(**_A							)
  spm.Load(str(_A							)							)
  return spm
def  lowerCamelCase__			(		_A							):
  with open(_A					,     'r'							) as f:
    return json.load(_A							)

def  lowerCamelCase__			(		_A					,     _A							):
  with open(_A					,     'w'							) as f:
    json.dump(_A					,     _A					,     indent=2							)