





'''simple docstring'''





import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
SCREAMING_SNAKE_CASE__      					=     "src/transformers"


# This is to make sure the transformers module imported is the one in the repo.
SCREAMING_SNAKE_CASE__      					=     direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
SCREAMING_SNAKE_CASE__      					=     re.compile(R"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
SCREAMING_SNAKE_CASE__      					=     re.compile(R"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
SCREAMING_SNAKE_CASE__      					=     re.compile(R"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
SCREAMING_SNAKE_CASE__      					=     [
    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),
    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),
    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),
    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),
    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),
    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),
    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),
    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),
    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),
    (
        "zero-shot-object-detection",
        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",
        "AutoModelForZeroShotObjectDetection",
    ),
    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),
    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),
    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),
    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),
    (
        "table-question-answering",
        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForTableQuestionAnswering",
    ),
    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),
    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),
    (
        "next-sentence-prediction",
        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",
        "AutoModelForNextSentencePrediction",
    ),
    (
        "audio-frame-classification",
        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForAudioFrameClassification",
    ),
    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),
    (
        "document-question-answering",
        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForDocumentQuestionAnswering",
    ),
    (
        "visual-question-answering",
        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",
        "AutoModelForVisualQuestionAnswering",
    ),
    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),
    (
        "zero-shot-image-classification",
        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",
        "AutoModelForZeroShotImageClassification",
    ),
    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),
    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),
    ("mask-generation", "MODEL_FOR_MASK_GENERATION_MAPPING_NAMES", "AutoModelForMaskGeneration"),
]




def        lowerCamelCase  (				_snake_case  :						Dict       ):





    '''simple docstring'''


    lowercase__		       =						re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)"     ,_snake_case       )
    return [m.group(0       ) for m in matches]




def        lowerCamelCase  (				):





    '''simple docstring'''


    lowercase__		       =						transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
    lowercase__		       =						{
        config.replace("Config"     ,""       ): model_type for model_type, config in config_maping_names.items()
    }

    # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
    lowercase__		       =						collections.defaultdict(_snake_case       )
    lowercase__		       =						collections.defaultdict(_snake_case       )
    lowercase__		       =						collections.defaultdict(_snake_case       )

    # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
    for attr_name in dir(_snake_case       ):
        lowercase__		       =						None
        if _re_tf_models.match(_snake_case       ) is not None:
            lowercase__		       =						tf_models
            lowercase__		       =						_re_tf_models.match(_snake_case       ).groups()[0]
        elif _re_flax_models.match(_snake_case       ) is not None:
            lowercase__		       =						flax_models
            lowercase__		       =						_re_flax_models.match(_snake_case       ).groups()[0]
        elif _re_pt_models.match(_snake_case       ) is not None:
            lowercase__		       =						pt_models
            lowercase__		       =						_re_pt_models.match(_snake_case       ).groups()[0]

        if lookup_dict is not None:
            while len(_snake_case       ) > 0:
                if attr_name in model_prefix_to_model_type:
                    lowercase__		       =						True
                    break
                # Try again after removing the last word in the name
                lowercase__		       =						"".join(camel_case_split(_snake_case       )[:-1]       )

    lowercase__		       =						set(list(pt_models.keys()       ) + list(tf_models.keys()       ) + list(flax_models.keys()       )       )
    lowercase__		       =						list(_snake_case       )
    all_models.sort()

    lowercase__		       =						{"model_type": all_models}
    lowercase__		       =						[pt_models[t] for t in all_models]
    lowercase__		       =						[tf_models[t] for t in all_models]
    lowercase__		       =						[flax_models[t] for t in all_models]

    # Now let's use the auto-mapping names to make sure
    lowercase__		       =						{}
    for t in all_models:
        if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
            lowercase__		       =						"AutoProcessor"
        elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
            lowercase__		       =						"AutoTokenizer"
        elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
            lowercase__		       =						"AutoFeatureExtractor"
        else:
            # Default to AutoTokenizer if a model has nothing, for backward compatibility.
            lowercase__		       =						"AutoTokenizer"

    lowercase__		       =						[processors[t] for t in all_models]

    return pd.DataFrame(_snake_case       )




def        lowerCamelCase  (				_snake_case  :						List[str]       ):





    '''simple docstring'''


    lowercase__		       =						[
        transformers_module.models.auto.modeling_auto,
        transformers_module.models.auto.modeling_tf_auto,
        transformers_module.models.auto.modeling_flax_auto,
    ]
    for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
        lowercase__		       =						[model_mapping, f'''TF_{model_mapping}''', f'''FLAX_{model_mapping}''']
        lowercase__		       =						[auto_class, f'''TF_{auto_class}''', f'''Flax_{auto_class}''']
        # Loop through all three frameworks
        for module, cls, mapping in zip(_snake_case     ,_snake_case     ,_snake_case       ):
            # The type of pipeline may not exist in this framework
            if not hasattr(_snake_case     ,_snake_case       ):
                continue
            # First extract all model_names
            lowercase__		       =						[]
            for name in getattr(_snake_case     ,_snake_case       ).values():
                if isinstance(_snake_case     ,_snake_case       ):
                    model_names.append(_snake_case       )
                else:
                    model_names.extend(list(_snake_case       )       )

            # Add pipeline tag and auto model class for those models
            table.update({model_name: (pipeline_tag, cls) for model_name in model_names}       )

    return table




def        lowerCamelCase  (				_snake_case  :						str     ,_snake_case  :						str       ):





    '''simple docstring'''


    lowercase__		       =						get_frameworks_table()
    lowercase__		       =						Dataset.from_pandas(_snake_case       )

    lowercase__		       =						hf_hub_download(
        "huggingface/transformers-metadata"     ,"pipeline_tags.json"     ,repo_type="dataset"     ,token=_snake_case       )
    lowercase__		       =						Dataset.from_json(_snake_case       )
    lowercase__		       =						{
        tags_dataset[i]["model_class"]: (tags_dataset[i]["pipeline_tag"], tags_dataset[i]["auto_class"])
        for i in range(len(_snake_case       )       )
    }
    lowercase__		       =						update_pipeline_and_auto_class_table(_snake_case       )

    # Sort the model classes to avoid some nondeterministic updates to create false update commits.
    lowercase__		       =						sorted(table.keys()       )
    lowercase__		       =						pd.DataFrame(
        {
            "model_class": model_classes,
            "pipeline_tag": [table[m][0] for m in model_classes],
            "auto_class": [table[m][1] for m in model_classes],
        }       )
    lowercase__		       =						Dataset.from_pandas(_snake_case       )

    with tempfile.TemporaryDirectory() as tmp_dir:
        frameworks_dataset.to_json(os.path.join(_snake_case     ,"frameworks.json"       )       )
        tags_dataset.to_json(os.path.join(_snake_case     ,"pipeline_tags.json"       )       )

        if commit_sha is not None:
            lowercase__		       =						(
                f'''Update with commit {commit_sha}\n\nSee: '''
                f'''https://github.com/huggingface/transformers/commit/{commit_sha}'''
            )
        else:
            lowercase__		       =						"Update"

        upload_folder(
            repo_id="huggingface/transformers-metadata"     ,folder_path=_snake_case     ,repo_type="dataset"     ,token=_snake_case     ,commit_message=_snake_case     ,)




def        lowerCamelCase  (				):





    '''simple docstring'''


    lowercase__		       =						{tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
    lowercase__		       =						transformers_module.pipelines.SUPPORTED_TASKS
    lowercase__		       =						[]
    for key in pipeline_tasks:
        if key not in in_table:
            lowercase__		       =						pipeline_tasks[key]["pt"]
            if isinstance(_snake_case     ,(list, tuple)       ):
                lowercase__		       =						model[0]
            lowercase__		       =						model.__name__
            if model not in in_table.values():
                missing.append(_snake_case       )

    if len(_snake_case       ) > 0:
        lowercase__		       =						", ".join(_snake_case       )
        raise ValueError(
            "The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside "
            f'''`utils/update_metadata.py`: {msg}. Please add them!'''       )


if __name__ == "__main__":
 SCREAMING_SNAKE_CASE__      					=     argparse.ArgumentParser()
 parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")
 parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")
 parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")
 SCREAMING_SNAKE_CASE__      					=     parser.parse_args()

 if args.check_only:
  check_pipeline_tags()
 else:
  update_metadata(args.token, args.commit_sha)









'''simple docstring'''





import json
import os
from pathlib import Path

import pytest

from datasets.download.download_config import DownloadConfig
from datasets.download.download_manager import DownloadManager
from datasets.utils.file_utils import hash_url_to_filename


SCREAMING_SNAKE_CASE__      					=     "http://www.mocksite.com/file1.txt"
SCREAMING_SNAKE_CASE__      					=     "\"text\": [\"foo\", \"foo\"]"
SCREAMING_SNAKE_CASE__      					=     "6d8ce9aa78a471c7477201efbeabd3bb01ac2e7d100a6dc024ba1608361f90a8"


class 				snake_case :
       lowerCAmelCase__  :Union[str, Any]						   =					200
       lowerCAmelCase__  :Tuple						   =					{"Content-Length": "100"}
       lowerCAmelCase__  :Any						   =					{}



       def 							_a      (				self     ,**UpperCAmelCase_      )  ->							List[Any]:
           return [bytes(UpperCAmelCase_     ,"utf-8"      )]





def        lowerCamelCase  (				*_snake_case  :						int     ,**_snake_case  :						int       ):





    '''simple docstring'''


    return MockResponse()




@pytest.mark.parametrize("urls_type"     ,[str, list, dict]       )
def        lowerCamelCase  (				_snake_case  :						str     ,_snake_case  :						List[Any]     ,_snake_case  :						List[Any]       ):





    '''simple docstring'''


    import requests

    monkeypatch.setattr(_snake_case     ,"request"     ,_snake_case       )

    lowercase__		       =						URL
    if issubclass(_snake_case     ,_snake_case       ):
        lowercase__		       =						url
    elif issubclass(_snake_case     ,_snake_case       ):
        lowercase__		       =						[url]
    elif issubclass(_snake_case     ,_snake_case       ):
        lowercase__		       =						{"train": url}
    lowercase__		       =						"dummy"
    lowercase__		       =						"downloads"
    lowercase__		       =						tmp_path
    lowercase__		       =						DownloadConfig(
        cache_dir=os.path.join(_snake_case     ,_snake_case       )     ,use_etag=_snake_case     ,)
    lowercase__		       =						DownloadManager(dataset_name=_snake_case     ,download_config=_snake_case       )
    lowercase__		       =						dl_manager.download(_snake_case       )
    lowercase__		       =						urls
    for downloaded_paths in [downloaded_paths]:
        if isinstance(_snake_case     ,_snake_case       ):
            lowercase__		       =						[downloaded_paths]
            lowercase__		       =						[urls]
        elif isinstance(_snake_case     ,_snake_case       ):
            assert "train" in downloaded_paths.keys()
            lowercase__		       =						downloaded_paths.values()
            lowercase__		       =						urls.values()
        assert downloaded_paths
        for downloaded_path, input_url in zip(_snake_case     ,_snake_case       ):
            assert downloaded_path == dl_manager.downloaded_paths[input_url]
            lowercase__		       =						Path(_snake_case       )
            lowercase__		       =						downloaded_path.parts
            assert parts[-1] == HASH
            assert parts[-2] == cache_subdir
            assert downloaded_path.exists()
            lowercase__		       =						downloaded_path.read_text()
            assert content == CONTENT
            lowercase__		       =						downloaded_path.with_suffix(".json"       )
            assert metadata_downloaded_path.exists()
            lowercase__		       =						json.loads(metadata_downloaded_path.read_text()       )
            assert metadata_content == {"url": URL, "etag": None}




@pytest.mark.parametrize("paths_type"     ,[str, list, dict]       )
def        lowerCamelCase  (				_snake_case  :						Optional[Any]     ,_snake_case  :						List[str]     ,_snake_case  :						List[str]       ):





    '''simple docstring'''


    lowercase__		       =						str(_snake_case       )
    if issubclass(_snake_case     ,_snake_case       ):
        lowercase__		       =						filename
    elif issubclass(_snake_case     ,_snake_case       ):
        lowercase__		       =						[filename]
    elif issubclass(_snake_case     ,_snake_case       ):
        lowercase__		       =						{"train": filename}
    lowercase__		       =						"dummy"
    lowercase__		       =						xz_file.parent
    lowercase__		       =						"extracted"
    lowercase__		       =						DownloadConfig(
        cache_dir=_snake_case     ,use_etag=_snake_case     ,)
    lowercase__		       =						DownloadManager(dataset_name=_snake_case     ,download_config=_snake_case       )
    lowercase__		       =						dl_manager.extract(_snake_case       )
    lowercase__		       =						paths
    for extracted_paths in [extracted_paths]:
        if isinstance(_snake_case     ,_snake_case       ):
            lowercase__		       =						[extracted_paths]
            lowercase__		       =						[paths]
        elif isinstance(_snake_case     ,_snake_case       ):
            assert "train" in extracted_paths.keys()
            lowercase__		       =						extracted_paths.values()
            lowercase__		       =						paths.values()
        assert extracted_paths
        for extracted_path, input_path in zip(_snake_case     ,_snake_case       ):
            assert extracted_path == dl_manager.extracted_paths[input_path]
            lowercase__		       =						Path(_snake_case       )
            lowercase__		       =						extracted_path.parts
            assert parts[-1] == hash_url_to_filename(_snake_case     ,etag=_snake_case       )
            assert parts[-2] == extracted_subdir
            assert extracted_path.exists()
            lowercase__		       =						extracted_path.read_text()
            lowercase__		       =						text_file.read_text()
            assert extracted_file_content == expected_file_content




def        lowerCamelCase  (				_snake_case  :						Tuple     ,_snake_case  :						List[str]       ):





    '''simple docstring'''


    assert path.endswith(".jsonl"       )
    for num_items, line in enumerate(_snake_case     ,start=1       ):
        lowercase__		       =						json.loads(line.decode("utf-8"       )       )
        assert item.keys() == {"col_1", "col_2", "col_3"}
    assert num_items == 4




@pytest.mark.parametrize("archive_jsonl"     ,["tar_jsonl_path", "zip_jsonl_path"]       )
def        lowerCamelCase  (				_snake_case  :						Any     ,_snake_case  :						Union[str, Any]       ):





    '''simple docstring'''


    lowercase__		       =						request.getfixturevalue(_snake_case       )
    lowercase__		       =						DownloadManager()
    for num_jsonl, (path, file) in enumerate(dl_manager.iter_archive(_snake_case       )     ,start=1       ):
        _test_jsonl(_snake_case     ,_snake_case       )
    assert num_jsonl == 2




@pytest.mark.parametrize("archive_nested_jsonl"     ,["tar_nested_jsonl_path", "zip_nested_jsonl_path"]       )
def        lowerCamelCase  (				_snake_case  :						Optional[Any]     ,_snake_case  :						Dict       ):





    '''simple docstring'''


    lowercase__		       =						request.getfixturevalue(_snake_case       )
    lowercase__		       =						DownloadManager()
    for num_tar, (path, file) in enumerate(dl_manager.iter_archive(_snake_case       )     ,start=1       ):
        for num_jsonl, (subpath, subfile) in enumerate(dl_manager.iter_archive(_snake_case       )     ,start=1       ):
            _test_jsonl(_snake_case     ,_snake_case       )
    assert num_tar == 1
    assert num_jsonl == 2




def        lowerCamelCase  (				_snake_case  :						int       ):





    '''simple docstring'''


    lowercase__		       =						DownloadManager()
    for num_file, file in enumerate(dl_manager.iter_files(_snake_case       )     ,start=1       ):
        assert os.path.basename(_snake_case       ) == ("test.txt" if num_file == 1 else "train.txt")
    assert num_file == 2




