





"""simple docstring"""





import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
SCREAMING_SNAKE_CASE__   =  "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
SCREAMING_SNAKE_CASE__   =  direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
SCREAMING_SNAKE_CASE__   =  re.compile(r"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
SCREAMING_SNAKE_CASE__   =  re.compile(r"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
SCREAMING_SNAKE_CASE__   =  re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
SCREAMING_SNAKE_CASE__   =  [
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]


def      UpperCAmelCase__				(    SCREAMING_SNAKE_CASE	:		Dict     ):







      '''simple docstring'''




      lowerCAmelCase    		=			re.finditer(""".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"""				,		SCREAMING_SNAKE_CASE     )
      return [m.group(0     ) for m in matches]


def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
      lowerCAmelCase    		=			{
          config.replace("""Config"""				,		""""""     ): model_type for model_type, config in config_maping_names.items()
      }

      # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
      lowerCAmelCase    		=			collections.defaultdict(SCREAMING_SNAKE_CASE     )
      lowerCAmelCase    		=			collections.defaultdict(SCREAMING_SNAKE_CASE     )
      lowerCAmelCase    		=			collections.defaultdict(SCREAMING_SNAKE_CASE     )

      # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
      for attr_name in dir(SCREAMING_SNAKE_CASE     ):
            lowerCAmelCase    		=			None
            if _re_tf_models.match(SCREAMING_SNAKE_CASE     ) is not None:
                  lowerCAmelCase    		=			tf_models
                  lowerCAmelCase    		=			_re_tf_models.match(SCREAMING_SNAKE_CASE     ).groups()[0]
            elif _re_flax_models.match(SCREAMING_SNAKE_CASE     ) is not None:
                  lowerCAmelCase    		=			flax_models
                  lowerCAmelCase    		=			_re_flax_models.match(SCREAMING_SNAKE_CASE     ).groups()[0]
            elif _re_pt_models.match(SCREAMING_SNAKE_CASE     ) is not None:
                  lowerCAmelCase    		=			pt_models
                  lowerCAmelCase    		=			_re_pt_models.match(SCREAMING_SNAKE_CASE     ).groups()[0]

            if lookup_dict is not None:
                  while len(SCREAMING_SNAKE_CASE     ) > 0:
                        if attr_name in model_prefix_to_model_type:
                              lowerCAmelCase    		=			True
                              break
                        # Try again after removing the last word in the name
                        lowerCAmelCase    		=			"""""".join(camel_case_split(SCREAMING_SNAKE_CASE     )[:-1]     )

      lowerCAmelCase    		=			set(list(pt_models.keys()     ) + list(tf_models.keys()     ) + list(flax_models.keys()     )     )
      lowerCAmelCase    		=			list(SCREAMING_SNAKE_CASE     )
      all_models.sort()

      lowerCAmelCase    		=			{"""model_type""": all_models}
      lowerCAmelCase    		=			[pt_models[t] for t in all_models]
      lowerCAmelCase    		=			[tf_models[t] for t in all_models]
      lowerCAmelCase    		=			[flax_models[t] for t in all_models]

      # Now let's use the auto-mapping names to make sure
      lowerCAmelCase    		=			{}
      for t in all_models:
            if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
                  lowerCAmelCase    		=			"""AutoProcessor"""
            elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
                  lowerCAmelCase    		=			"""AutoTokenizer"""
            elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
                  lowerCAmelCase    		=			"""AutoFeatureExtractor"""
            else:
                  # Default to AutoTokenizer if a model has nothing, for backward compatibility.
                  lowerCAmelCase    		=			"""AutoTokenizer"""

      lowerCAmelCase    		=			[processors[t] for t in all_models]

      return pd.DataFrame(SCREAMING_SNAKE_CASE     )


def      UpperCAmelCase__				(    SCREAMING_SNAKE_CASE	:		List[str]     ):







      '''simple docstring'''




      lowerCAmelCase    		=			[
          transformers_module.models.auto.modeling_auto,
          transformers_module.models.auto.modeling_tf_auto,
          transformers_module.models.auto.modeling_flax_auto,
      ]
      for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
            lowerCAmelCase    		=			[model_mapping, F'TF_{model_mapping}', F'FLAX_{model_mapping}']
            lowerCAmelCase    		=			[auto_class, F'TF_{auto_class}', F'Flax_{auto_class}']
            # Loop through all three frameworks
            for module, cls, mapping in zip(SCREAMING_SNAKE_CASE				,		SCREAMING_SNAKE_CASE				,		SCREAMING_SNAKE_CASE     ):
                  # The type of pipeline may not exist in this framework
                  if not hasattr(SCREAMING_SNAKE_CASE				,		SCREAMING_SNAKE_CASE     ):
                        continue
                  # First extract all model_names
                  lowerCAmelCase    		=			[]
                  for name in getattr(SCREAMING_SNAKE_CASE				,		SCREAMING_SNAKE_CASE     ).values():
                        if isinstance(SCREAMING_SNAKE_CASE				,		SCREAMING_SNAKE_CASE     ):
                              model_names.append(SCREAMING_SNAKE_CASE     )
                        else:
                              model_names.extend(list(SCREAMING_SNAKE_CASE     )     )

            # Add pipeline tag and auto model class for those models
                  table.update({model_name: (pipeline_tag, cls) for model_name in model_names}     )

      return table


def      UpperCAmelCase__				(    SCREAMING_SNAKE_CASE	:		Union[str, Any]				,		SCREAMING_SNAKE_CASE	:		Union[str, Any]     ):







      '''simple docstring'''




      lowerCAmelCase    		=			get_frameworks_table()
      lowerCAmelCase    		=			Dataset.from_pandas(SCREAMING_SNAKE_CASE     )

      lowerCAmelCase    		=			hf_hub_download(
          """huggingface/transformers-metadata"""				,		"""pipeline_tags.json"""				,		repo_type="""dataset"""				,		token=SCREAMING_SNAKE_CASE     )
      lowerCAmelCase    		=			Dataset.from_json(SCREAMING_SNAKE_CASE     )
      lowerCAmelCase    		=			{
          tags_dataset[i]["""model_class"""]: (tags_dataset[i]["""pipeline_tag"""], tags_dataset[i]["""auto_class"""])
          for i in range(len(SCREAMING_SNAKE_CASE     )     )
      }
      lowerCAmelCase    		=			update_pipeline_and_auto_class_table(SCREAMING_SNAKE_CASE     )

      # Sort the model classes to avoid some nondeterministic updates to create false update commits.
      lowerCAmelCase    		=			sorted(table.keys()     )
      lowerCAmelCase    		=			pd.DataFrame(
          {
              """model_class""": model_classes,
              """pipeline_tag""": [table[m][0] for m in model_classes],
              """auto_class""": [table[m][1] for m in model_classes],
          }     )
      lowerCAmelCase    		=			Dataset.from_pandas(SCREAMING_SNAKE_CASE     )

      with tempfile.TemporaryDirectory() as tmp_dir:
            frameworks_dataset.to_json(os.path.join(SCREAMING_SNAKE_CASE				,		"""frameworks.json"""     )     )
            tags_dataset.to_json(os.path.join(SCREAMING_SNAKE_CASE				,		"""pipeline_tags.json"""     )     )

            if commit_sha is not None:
                  lowerCAmelCase    		=			(
                      F'Update with commit {commit_sha}\n\nSee: '
                      F'https://github.com/huggingface/transformers/commit/{commit_sha}'
                  )
            else:
                  lowerCAmelCase    		=			"""Update"""

            upload_folder(
                repo_id="""huggingface/transformers-metadata"""				,		folder_path=SCREAMING_SNAKE_CASE				,		repo_type="""dataset"""				,		token=SCREAMING_SNAKE_CASE				,		commit_message=SCREAMING_SNAKE_CASE				,		)


def      UpperCAmelCase__				(    ):







      '''simple docstring'''




      lowerCAmelCase    		=			{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
      lowerCAmelCase    		=			transformers_module.pipelines.SUPPORTED_TASKS
      lowerCAmelCase    		=			[]
      for key in pipeline_tasks:
            if key not in in_table:
                  lowerCAmelCase    		=			pipeline_tasks[key]["""pt"""]
                  if isinstance(SCREAMING_SNAKE_CASE				,		(list, tuple)     ):
                        lowerCAmelCase    		=			model[0]
                  lowerCAmelCase    		=			model.__name__
                  if model not in in_table.values():
                        missing.append(SCREAMING_SNAKE_CASE     )

      if len(SCREAMING_SNAKE_CASE     ) > 0:
            lowerCAmelCase    		=			""", """.join(SCREAMING_SNAKE_CASE     )
            raise ValueError(
                """The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside """
                F'`utils/update_metadata.py`: {msg}. Please add them!'     )


if __name__ == "__main__":
       SCREAMING_SNAKE_CASE__   =  argparse.ArgumentParser()
       parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
       parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
       parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
       SCREAMING_SNAKE_CASE__   =  parser.parse_args()

       if args.check_only:
              check_pipeline_tags()
       else:
              update_metadata(args.token, args.commit_sha)













"""simple docstring"""





import io
import json
import unittest

from parameterized import parameterized

from transformers import FSMTForConditionalGeneration, FSMTTokenizer
from transformers.testing_utils import get_tests_dir, require_torch, slow, torch_device
from utils import calculate_bleu


SCREAMING_SNAKE_CASE__   =  get_tests_dir() + "/test_data/fsmt/fsmt_val_data.json"
with io.open(filename, "r", encoding="utf-8") as f:
       SCREAMING_SNAKE_CASE__   =  json.load(f)

@require_torch
class 						lowercase   (   unittest.TestCase   ):





      def        _snake_case     (			self ,		lowercase					)						->	Tuple:
            return FSMTTokenizer.from_pretrained(lowercase					)





      def        _snake_case     (			self ,		lowercase					)						->	Dict:
            lowerCAmelCase    		=			FSMTForConditionalGeneration.from_pretrained(lowercase					).to(lowercase					)
            if torch_device == "cuda":
                  model.half()
            return model





      @parameterized.expand(
          [
              ["""en-ru""", 26.0],
              ["""ru-en""", 22.0],
              ["""en-de""", 22.0],
              ["""de-en""", 29.0],
          ]					)
      @slow
      def        _snake_case     (			self ,		lowercase ,		lowercase					)						->	Dict:
            # note: this test is not testing the best performance since it only evals a small batch
            # but it should be enough to detect a regression in the output quality
            lowerCAmelCase    		=			f'facebook/wmt19-{pair}'
            lowerCAmelCase    		=			self.get_tokenizer(lowercase					)
            lowerCAmelCase    		=			self.get_model(lowercase					)

            lowerCAmelCase    		=			bleu_data[pair]["""src"""]
            lowerCAmelCase    		=			bleu_data[pair]["""tgt"""]

            lowerCAmelCase    		=			tokenizer(lowercase ,		return_tensors="""pt""" ,		truncation=lowercase ,		padding="""longest"""					).to(lowercase					)
            lowerCAmelCase    		=			model.generate(
                input_ids=batch.input_ids ,		num_beams=8 ,		)
            lowerCAmelCase    		=			tokenizer.batch_decode(
                lowercase ,		skip_special_tokens=lowercase ,		clean_up_tokenization_spaces=lowercase					)
            lowerCAmelCase    		=			calculate_bleu(lowercase ,		lowercase					)
            print(lowercase					)
            self.assertGreaterEqual(scores["""bleu"""] ,		lowercase					)






