


import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
UpperCAmelCase    =							"""src/transformers"""


# This is to make sure the transformers module imported is the one in the repo.
UpperCAmelCase    =							direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
UpperCAmelCase    =							re.compile(r"""TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
UpperCAmelCase    =							re.compile(r"""Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
UpperCAmelCase    =							re.compile(r"""(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)""")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
UpperCAmelCase    =							[
    ("""pretraining""", """MODEL_FOR_PRETRAINING_MAPPING_NAMES""", """AutoModelForPreTraining"""),
    ("""feature-extraction""", """MODEL_MAPPING_NAMES""", """AutoModel"""),
    ("""audio-classification""", """MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForAudioClassification"""),
    ("""text-generation""", """MODEL_FOR_CAUSAL_LM_MAPPING_NAMES""", """AutoModelForCausalLM"""),
    ("""automatic-speech-recognition""", """MODEL_FOR_CTC_MAPPING_NAMES""", """AutoModelForCTC"""),
    ("""image-classification""", """MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForImageClassification"""),
    ("""image-segmentation""", """MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES""", """AutoModelForImageSegmentation"""),
    ("""fill-mask""", """MODEL_FOR_MASKED_LM_MAPPING_NAMES""", """AutoModelForMaskedLM"""),
    ("""object-detection""", """MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES""", """AutoModelForObjectDetection"""),
    (
        """zero-shot-object-detection""",
        """MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES""",
        """AutoModelForZeroShotObjectDetection""",
    ),
    ("""question-answering""", """MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES""", """AutoModelForQuestionAnswering"""),
    ("""text2text-generation""", """MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES""", """AutoModelForSeq2SeqLM"""),
    ("""text-classification""", """MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForSequenceClassification"""),
    ("""automatic-speech-recognition""", """MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES""", """AutoModelForSpeechSeq2Seq"""),
    (
        """table-question-answering""",
        """MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForTableQuestionAnswering""",
    ),
    ("""token-classification""", """MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForTokenClassification"""),
    ("""multiple-choice""", """MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES""", """AutoModelForMultipleChoice"""),
    (
        """next-sentence-prediction""",
        """MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES""",
        """AutoModelForNextSentencePrediction""",
    ),
    (
        """audio-frame-classification""",
        """MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES""",
        """AutoModelForAudioFrameClassification""",
    ),
    ("""audio-xvector""", """MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES""", """AutoModelForAudioXVector"""),
    (
        """document-question-answering""",
        """MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForDocumentQuestionAnswering""",
    ),
    (
        """visual-question-answering""",
        """MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES""",
        """AutoModelForVisualQuestionAnswering""",
    ),
    ("""image-to-text""", """MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES""", """AutoModelForVision2Seq"""),
    (
        """zero-shot-image-classification""",
        """MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES""",
        """AutoModelForZeroShotImageClassification""",
    ),
    ("""depth-estimation""", """MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES""", """AutoModelForDepthEstimation"""),
    ("""video-classification""", """MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES""", """AutoModelForVideoClassification"""),
    ("""mask-generation""", """MODEL_FOR_MASK_GENERATION_MAPPING_NAMES""", """AutoModelForMaskGeneration"""),
]






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      )->						Optional[int]:




      """simple docstring"""

      snake_case_ =   re.finditer('''.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'''      ,      SCREAMING_SNAKE_CASE      )
      return [m.group(0      ) for m in matches]






def  __lowerCAmelCase					()->						List[str]:




      """simple docstring"""

      snake_case_ =   transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
      snake_case_ =   {
          config.replace('''Config'''      ,      ''''''      ): model_type for model_type, config in config_maping_names.items()
      }

      # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
      snake_case_ =   collections.defaultdict(SCREAMING_SNAKE_CASE      )
      snake_case_ =   collections.defaultdict(SCREAMING_SNAKE_CASE      )
      snake_case_ =   collections.defaultdict(SCREAMING_SNAKE_CASE      )

      # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
      for attr_name in dir(SCREAMING_SNAKE_CASE      ):
            snake_case_ =   None
            if _re_tf_models.match(SCREAMING_SNAKE_CASE      ) is not None:
                  snake_case_ =   tf_models
                  snake_case_ =   _re_tf_models.match(SCREAMING_SNAKE_CASE      ).groups()[0]
            elif _re_flax_models.match(SCREAMING_SNAKE_CASE      ) is not None:
                  snake_case_ =   flax_models
                  snake_case_ =   _re_flax_models.match(SCREAMING_SNAKE_CASE      ).groups()[0]
            elif _re_pt_models.match(SCREAMING_SNAKE_CASE      ) is not None:
                  snake_case_ =   pt_models
                  snake_case_ =   _re_pt_models.match(SCREAMING_SNAKE_CASE      ).groups()[0]

            if lookup_dict is not None:
                  while len(SCREAMING_SNAKE_CASE      ) > 0:
                        if attr_name in model_prefix_to_model_type:
                              snake_case_ =   True
                              break
                        # Try again after removing the last word in the name
                        snake_case_ =   ''''''.join(camel_case_split(SCREAMING_SNAKE_CASE      )[:-1]      )

      snake_case_ =   set(list(pt_models.keys()      ) + list(tf_models.keys()      ) + list(flax_models.keys()      )      )
      snake_case_ =   list(SCREAMING_SNAKE_CASE      )
      all_models.sort()

      snake_case_ =   {'''model_type''': all_models}
      snake_case_ =   [pt_models[t] for t in all_models]
      snake_case_ =   [tf_models[t] for t in all_models]
      snake_case_ =   [flax_models[t] for t in all_models]

      # Now let's use the auto-mapping names to make sure
      snake_case_ =   {}
      for t in all_models:
            if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
                  snake_case_ =   '''AutoProcessor'''
            elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
                  snake_case_ =   '''AutoTokenizer'''
            elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
                  snake_case_ =   '''AutoFeatureExtractor'''
            else:
                  # Default to AutoTokenizer if a model has nothing, for backward compatibility.
                  snake_case_ =   '''AutoTokenizer'''

      snake_case_ =   [processors[t] for t in all_models]

      return pd.DataFrame(SCREAMING_SNAKE_CASE      )






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      )->						Optional[int]:




      """simple docstring"""

      snake_case_ =   [
          transformers_module.models.auto.modeling_auto,
          transformers_module.models.auto.modeling_tf_auto,
          transformers_module.models.auto.modeling_flax_auto,
      ]
      for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
            snake_case_ =   [model_mapping, f'''TF_{model_mapping}''', f'''FLAX_{model_mapping}''']
            snake_case_ =   [auto_class, f'''TF_{auto_class}''', f'''Flax_{auto_class}''']
            # Loop through all three frameworks
            for module, cls, mapping in zip(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ):
                  # The type of pipeline may not exist in this framework
                  if not hasattr(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ):
                        continue
                  # First extract all model_names
                  snake_case_ =   []
                  for name in getattr(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ).values():
                        if isinstance(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      ):
                              model_names.append(SCREAMING_SNAKE_CASE      )
                        else:
                              model_names.extend(list(SCREAMING_SNAKE_CASE      )      )

            # Add pipeline tag and auto model class for those models
                  table.update({model_name: (pipeline_tag, cls) for model_name in model_names}      )

      return table






def  __lowerCAmelCase					(SCREAMING_SNAKE_CASE      ,      SCREAMING_SNAKE_CASE      )->						Optional[Any]:




      """simple docstring"""

      snake_case_ =   get_frameworks_table()
      snake_case_ =   Dataset.from_pandas(SCREAMING_SNAKE_CASE      )

      snake_case_ =   hf_hub_download(
          '''huggingface/transformers-metadata'''      ,      '''pipeline_tags.json'''      ,      repo_type='''dataset'''      ,      token=SCREAMING_SNAKE_CASE      )
      snake_case_ =   Dataset.from_json(SCREAMING_SNAKE_CASE      )
      snake_case_ =   {
          tags_dataset[i]['''model_class''']: (tags_dataset[i]['''pipeline_tag'''], tags_dataset[i]['''auto_class'''])
          for i in range(len(SCREAMING_SNAKE_CASE      )      )
      }
      snake_case_ =   update_pipeline_and_auto_class_table(SCREAMING_SNAKE_CASE      )

      # Sort the model classes to avoid some nondeterministic updates to create false update commits.
      snake_case_ =   sorted(table.keys()      )
      snake_case_ =   pd.DataFrame(
          {
              '''model_class''': model_classes,
              '''pipeline_tag''': [table[m][0] for m in model_classes],
              '''auto_class''': [table[m][1] for m in model_classes],
          }      )
      snake_case_ =   Dataset.from_pandas(SCREAMING_SNAKE_CASE      )

      with tempfile.TemporaryDirectory() as tmp_dir:
            frameworks_dataset.to_json(os.path.join(SCREAMING_SNAKE_CASE      ,      '''frameworks.json'''      )      )
            tags_dataset.to_json(os.path.join(SCREAMING_SNAKE_CASE      ,      '''pipeline_tags.json'''      )      )

            if commit_sha is not None:
                  snake_case_ =   (
                      f'''Update with commit {commit_sha}\n\nSee: '''
                      f'''https://github.com/huggingface/transformers/commit/{commit_sha}'''
                  )
            else:
                  snake_case_ =   '''Update'''

            upload_folder(
                repo_id='''huggingface/transformers-metadata'''      ,      folder_path=SCREAMING_SNAKE_CASE      ,      repo_type='''dataset'''      ,      token=SCREAMING_SNAKE_CASE      ,      commit_message=SCREAMING_SNAKE_CASE      ,      )






def  __lowerCAmelCase					()->						Any:




      """simple docstring"""

      snake_case_ =   {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
      snake_case_ =   transformers_module.pipelines.SUPPORTED_TASKS
      snake_case_ =   []
      for key in pipeline_tasks:
            if key not in in_table:
                  snake_case_ =   pipeline_tasks[key]['''pt''']
                  if isinstance(SCREAMING_SNAKE_CASE      ,      (list, tuple)      ):
                        snake_case_ =   model[0]
                  snake_case_ =   model.__name__
                  if model not in in_table.values():
                        missing.append(SCREAMING_SNAKE_CASE      )

      if len(SCREAMING_SNAKE_CASE      ) > 0:
            snake_case_ =   ''', '''.join(SCREAMING_SNAKE_CASE      )
            raise ValueError(
                '''The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '''
                f'''`utils/update_metadata.py`: {msg}. Please add them!'''      )


if __name__ == "__main__":
   UpperCAmelCase    =							argparse.ArgumentParser()
   parser.add_argument("""--token""", type=str, help="""The token to use to push to the transformers-metadata dataset.""")
   parser.add_argument("""--commit_sha""", type=str, help="""The sha of the commit going with this update.""")
   parser.add_argument("""--check-only""", action="""store_true""", help="""Activate to just check all pipelines are present.""")
   UpperCAmelCase    =							parser.parse_args()

   if args.check_only:
      check_pipeline_tags()
   else:
      update_metadata(args.token, args.commit_sha)



import unittest

from huggingface_hub import hf_hub_download

from transformers import MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING, VideoMAEFeatureExtractor
from transformers.pipelines import VideoClassificationPipeline, pipeline
from transformers.testing_utils import (
    is_pipeline_test,
    nested_simplify,
    require_decord,
    require_tf,
    require_torch,
    require_torch_or_tf,
    require_vision,
)

from .test_pipelines_common import ANY







@is_pipeline_test
@require_torch_or_tf
@require_vision
@require_decord
class lowerCAmelCase_		(				unittest.TestCase ):

 '''simple docstring'''

 __snake_case				=						MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase	,   _UpperCAmelCase      ):
       snake_case_ =   hf_hub_download(
           repo_id='''nateraw/video-demo'''	,   filename='''archery.mp4'''	,   repo_type='''dataset'''      )
       snake_case_ =   VideoClassificationPipeline(model=_UpperCAmelCase	,   image_processor=_UpperCAmelCase	,   top_k=2      )
       snake_case_ =   [
           example_video_filepath,
           '''https://huggingface.co/datasets/nateraw/video-demo/resolve/main/archery.mp4''',
       ]
       return video_classifier, examples



 def 		UpperCamelCase__  (			self	,   _UpperCAmelCase	,   _UpperCAmelCase      ):
       for example in examples:
             snake_case_ =   video_classifier(_UpperCAmelCase      )

             self.assertEqual(
                 _UpperCAmelCase	,   [
                     {'''score''': ANY(_UpperCAmelCase      ), '''label''': ANY(_UpperCAmelCase      )},
                     {'''score''': ANY(_UpperCAmelCase      ), '''label''': ANY(_UpperCAmelCase      )},
                 ]	,   )



 @require_torch
 def 		UpperCamelCase__  (			self      ):
       snake_case_ =   '''hf-internal-testing/tiny-random-VideoMAEForVideoClassification'''
       snake_case_ =   VideoMAEFeatureExtractor(
           size={'''shortest_edge''': 10}	,   crop_size={'''height''': 10, '''width''': 10}      )
       snake_case_ =   pipeline(
           '''video-classification'''	,   model=_UpperCAmelCase	,   feature_extractor=_UpperCAmelCase	,   frame_sampling_rate=4      )

       snake_case_ =   hf_hub_download(repo_id='''nateraw/video-demo'''	,   filename='''archery.mp4'''	,   repo_type='''dataset'''      )
       snake_case_ =   video_classifier(_UpperCAmelCase	,   top_k=2      )
       self.assertEqual(
           nested_simplify(_UpperCAmelCase	,   decimals=4      )	,   [{'''score''': 0.5_199, '''label''': '''LABEL_0'''}, {'''score''': 0.4_801, '''label''': '''LABEL_1'''}]	,   )

       snake_case_ =   video_classifier(
           [
               video_file_path,
               video_file_path,
           ]	,   top_k=2	,   )
       self.assertEqual(
           nested_simplify(_UpperCAmelCase	,   decimals=4      )	,   [
               [{'''score''': 0.5_199, '''label''': '''LABEL_0'''}, {'''score''': 0.4_801, '''label''': '''LABEL_1'''}],
               [{'''score''': 0.5_199, '''label''': '''LABEL_0'''}, {'''score''': 0.4_801, '''label''': '''LABEL_1'''}],
           ]	,   )





 @require_tf
 def 		UpperCamelCase__  (			self      ):
       pass