



'''simple docstring'''





import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
A__    :   Optional[int]           =					'''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
A__    :   List[Any]           =					direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
A__    :   Dict           =					re.compile(R'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
A__    :   List[str]           =					re.compile(R'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
A__    :   Union[str, Any]           =					re.compile(R'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
A__    :   Optional[int]           =					[
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]





def   a_   (		_UpperCAmelCase    :      Optional[Any]				)    ->  Dict:
      __snake_case :   Optional[Any]													=    re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)' ,_UpperCAmelCase				)
      return [m.group(0				) for m in matches]





def   a_   (		)    ->  Tuple:
      __snake_case :   str													=    transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
      __snake_case :   Any													=    {
          config.replace('Config' ,''				): model_type for model_type, config in config_maping_names.items()
      }

      # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
      __snake_case :   Optional[int]													=    collections.defaultdict(_UpperCAmelCase				)
      __snake_case :   List[Any]													=    collections.defaultdict(_UpperCAmelCase				)
      __snake_case :   Dict													=    collections.defaultdict(_UpperCAmelCase				)

      # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
      for attr_name in dir(_UpperCAmelCase				):
            __snake_case :   str													=    None
            if _re_tf_models.match(_UpperCAmelCase				) is not None:
                  __snake_case :   Tuple													=    tf_models
                  __snake_case :   Dict													=    _re_tf_models.match(_UpperCAmelCase				).groups()[0]
            elif _re_flax_models.match(_UpperCAmelCase				) is not None:
                  __snake_case :   int													=    flax_models
                  __snake_case :   Optional[int]													=    _re_flax_models.match(_UpperCAmelCase				).groups()[0]
            elif _re_pt_models.match(_UpperCAmelCase				) is not None:
                  __snake_case :   Optional[int]													=    pt_models
                  __snake_case :   Optional[int]													=    _re_pt_models.match(_UpperCAmelCase				).groups()[0]

            if lookup_dict is not None:
                  while len(_UpperCAmelCase				) > 0:
                        if attr_name in model_prefix_to_model_type:
                              __snake_case :   Any													=    True
                              break
                        # Try again after removing the last word in the name
                        __snake_case :   Dict													=    ''.join(camel_case_split(_UpperCAmelCase				)[:-1]				)

      __snake_case :   Union[str, Any]													=    set(list(pt_models.keys()				) + list(tf_models.keys()				) + list(flax_models.keys()				)				)
      __snake_case :   Optional[int]													=    list(_UpperCAmelCase				)
      all_models.sort()

      __snake_case :   Any													=    {'model_type': all_models}
      __snake_case :   Any													=    [pt_models[t] for t in all_models]
      __snake_case :   Any													=    [tf_models[t] for t in all_models]
      __snake_case :   int													=    [flax_models[t] for t in all_models]

      # Now let's use the auto-mapping names to make sure
      __snake_case :   Optional[Any]													=    {}
      for t in all_models:
            if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
                  __snake_case :   int													=    'AutoProcessor'
            elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
                  __snake_case :   Optional[int]													=    'AutoTokenizer'
            elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
                  __snake_case :   Dict													=    'AutoFeatureExtractor'
            else:
                  # Default to AutoTokenizer if a model has nothing, for backward compatibility.
                  __snake_case :   List[str]													=    'AutoTokenizer'

      __snake_case :   Union[str, Any]													=    [processors[t] for t in all_models]

      return pd.DataFrame(_UpperCAmelCase				)





def   a_   (		_UpperCAmelCase    :      Tuple				)    ->  Optional[Any]:
      __snake_case :   Dict													=    [
          transformers_module.models.auto.modeling_auto,
          transformers_module.models.auto.modeling_tf_auto,
          transformers_module.models.auto.modeling_flax_auto,
      ]
      for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
            __snake_case :   str													=    [model_mapping, f'''TF_{model_mapping}''', f'''FLAX_{model_mapping}''']
            __snake_case :   Tuple													=    [auto_class, f'''TF_{auto_class}''', f'''Flax_{auto_class}''']
            # Loop through all three frameworks
            for module, cls, mapping in zip(_UpperCAmelCase ,_UpperCAmelCase ,_UpperCAmelCase				):
                  # The type of pipeline may not exist in this framework
                  if not hasattr(_UpperCAmelCase ,_UpperCAmelCase				):
                        continue
                  # First extract all model_names
                  __snake_case :   List[Any]													=    []
                  for name in getattr(_UpperCAmelCase ,_UpperCAmelCase				).values():
                        if isinstance(_UpperCAmelCase ,_UpperCAmelCase				):
                              model_names.append(_UpperCAmelCase				)
                        else:
                              model_names.extend(list(_UpperCAmelCase				)				)

            # Add pipeline tag and auto model class for those models
                  table.update({model_name: (pipeline_tag, cls) for model_name in model_names}				)

      return table





def   a_   (		_UpperCAmelCase    :      Optional[Any] ,_UpperCAmelCase    :      Dict				)    ->  Dict:
      __snake_case :   Dict													=    get_frameworks_table()
      __snake_case :   List[str]													=    Dataset.from_pandas(_UpperCAmelCase				)

      __snake_case :   Union[str, Any]													=    hf_hub_download(
          'huggingface/transformers-metadata' ,'pipeline_tags.json' ,repo_type='dataset' ,token=_UpperCAmelCase				)
      __snake_case :   Optional[Any]													=    Dataset.from_json(_UpperCAmelCase				)
      __snake_case :   Dict													=    {
          tags_dataset[i]['model_class']: (tags_dataset[i]['pipeline_tag'], tags_dataset[i]['auto_class'])
          for i in range(len(_UpperCAmelCase				)				)
      }
      __snake_case :   Optional[Any]													=    update_pipeline_and_auto_class_table(_UpperCAmelCase				)

      # Sort the model classes to avoid some nondeterministic updates to create false update commits.
      __snake_case :   Tuple													=    sorted(table.keys()				)
      __snake_case :   Union[str, Any]													=    pd.DataFrame(
          {
              'model_class': model_classes,
              'pipeline_tag': [table[m][0] for m in model_classes],
              'auto_class': [table[m][1] for m in model_classes],
          }				)
      __snake_case :   Tuple													=    Dataset.from_pandas(_UpperCAmelCase				)

      with tempfile.TemporaryDirectory() as tmp_dir:
            frameworks_dataset.to_json(os.path.join(_UpperCAmelCase ,'frameworks.json'				)				)
            tags_dataset.to_json(os.path.join(_UpperCAmelCase ,'pipeline_tags.json'				)				)

            if commit_sha is not None:
                  __snake_case :   Optional[Any]													=    (
                      f'''Update with commit {commit_sha}\n\nSee: '''
                      f'''https://github.com/huggingface/transformers/commit/{commit_sha}'''
                  )
            else:
                  __snake_case :   Optional[Any]													=    'Update'

            upload_folder(
                repo_id='huggingface/transformers-metadata' ,folder_path=_UpperCAmelCase ,repo_type='dataset' ,token=_UpperCAmelCase ,commit_message=_UpperCAmelCase ,)





def   a_   (		)    ->  List[str]:
      __snake_case :   str													=    {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
      __snake_case :   Optional[Any]													=    transformers_module.pipelines.SUPPORTED_TASKS
      __snake_case :   Dict													=    []
      for key in pipeline_tasks:
            if key not in in_table:
                  __snake_case :   Tuple													=    pipeline_tasks[key]['pt']
                  if isinstance(_UpperCAmelCase ,(list, tuple)				):
                        __snake_case :   Tuple													=    model[0]
                  __snake_case :   Tuple													=    model.__name__
                  if model not in in_table.values():
                        missing.append(_UpperCAmelCase				)

      if len(_UpperCAmelCase				) > 0:
            __snake_case :   Optional[int]													=    ', '.join(_UpperCAmelCase				)
            raise ValueError(
                'The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '
                f'''`utils/update_metadata.py`: {msg}. Please add them!'''				)


if __name__ == "__main__":
    A__    :   Optional[Any]           =					argparse.ArgumentParser()
    parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
    parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
    parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
    A__    :   Union[str, Any]           =					parser.parse_args()

    if args.check_only:
        check_pipeline_tags()
    else:
        update_metadata(args.token, args.commit_sha)








'''simple docstring'''





import json
import os
import unittest

from transformers import DebertaTokenizer, DebertaTokenizerFast
from transformers.models.deberta.tokenization_deberta import VOCAB_FILES_NAMES
from transformers.testing_utils import slow

from ...test_tokenization_common import TokenizerTesterMixin



class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= DebertaTokenizer
 A__				= True
 A__				= DebertaTokenizerFast
 def A_   (   self						:							Optional[int]  )      ->   str:







       '''simple docstring'''
       super().setUp()

       # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
       __snake_case :   Union[str, Any]													=    [
           'l',
           'o',
           'w',
           'e',
           'r',
           's',
           't',
           'i',
           'd',
           'n',
           '\u0120',
           '\u0120l',
           '\u0120n',
           '\u0120lo',
           '\u0120low',
           'er',
           '\u0120lowest',
           '\u0120newer',
           '\u0120wider',
           '[UNK]',
       ]
       __snake_case :   List[str]													=    dict(zip(__a      ,     range(len(__a  )  )  )  )
       __snake_case :   int													=    ['#version: 0.2', '\u0120 l', '\u0120l o', '\u0120lo w', 'e r', '']
       __snake_case :   Any													=    {'unk_token': '[UNK]'}

       __snake_case :   Union[str, Any]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       __snake_case :   Dict													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['merges_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write(json.dumps(__a  ) + '\n'  )
       with open(self.merges_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write('\n'.join(__a  )  )
 def A_   (   self						:							Optional[int]      ,     **__a						:							List[Any]  )      ->   Optional[Any]:







       '''simple docstring'''
       kwargs.update(self.special_tokens_map  )
       return self.tokenizer_class.from_pretrained(self.tmpdirname      ,     **__a  )
 def A_   (   self						:							Optional[int]      ,     __a						:							Dict  )      ->   List[str]:







       '''simple docstring'''
       __snake_case :   Union[str, Any]													=    'lower newer'
       __snake_case :   List[Any]													=    'lower newer'
       return input_text, output_text
 def A_   (   self						:							Dict  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   str													=    self.get_tokenizer()
       __snake_case :   List[str]													=    'lower newer'
       __snake_case :   Tuple													=    ['l', 'o', 'w', 'er', '\u0120', 'n', 'e', 'w', 'er']
       __snake_case :   Optional[int]													=    tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   str													=    tokens + [tokenizer.unk_token]
       __snake_case :   Tuple													=    [0, 1, 2, 15, 10, 9, 3, 2, 15, 19]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     __a  )
 def A_   (   self						:							List[Any]  )      ->   Dict:







       '''simple docstring'''
       __snake_case :   List[str]													=    self.get_tokenizer()
       __snake_case :   List[str]													=    tokenizer('Hello'      ,     'World'  )
       __snake_case :   int													=    [0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]
       self.assertListEqual(tokd['token_type_ids']      ,     __a  )
 @slow
 def A_   (   self						:							Dict  )      ->   str:







       '''simple docstring'''
       __snake_case :   str													=    self.tokenizer_class.from_pretrained('microsoft/deberta-base'  )

       __snake_case :   str													=    tokenizer.encode('sequence builders'      ,     add_special_tokens=__a  )
       __snake_case :   Any													=    tokenizer.encode('multi-sequence build'      ,     add_special_tokens=__a  )

       __snake_case :   List[Any]													=    tokenizer.encode(
           'sequence builders'      ,     add_special_tokens=__a      ,     add_prefix_space=__a  )
       __snake_case :   Union[str, Any]													=    tokenizer.encode(
           'sequence builders'      ,     'multi-sequence build'      ,     add_special_tokens=__a      ,     add_prefix_space=__a  )

       __snake_case :   int													=    tokenizer.build_inputs_with_special_tokens(__a  )
       __snake_case :   Tuple													=    tokenizer.build_inputs_with_special_tokens(__a      ,     __a  )

       assert encoded_sentence == encoded_text_from_decode
       assert encoded_pair == encoded_pair_from_decode





 @slow
 def A_   (   self						:							Optional[Any]  )      ->   Optional[int]:







       '''simple docstring'''
       __snake_case :   int													=    [self.tokenizer_class]
       if self.test_rust_tokenizer:
             tokenizer_classes.append(self.rust_tokenizer_class  )

       for tokenizer_class in tokenizer_classes:
             __snake_case :   List[Any]													=    tokenizer_class.from_pretrained('microsoft/deberta-base'  )

             __snake_case :   int													=    [
                 'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                 'ALBERT incorporates two parameter reduction techniques',
                 'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                 ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                 ' vocabulary embedding.',
             ]

             __snake_case :   Union[str, Any]													=    tokenizer(__a      ,     padding=__a  )
             __snake_case :   List[str]													=    [tokenizer.decode(__a      ,     skip_special_tokens=__a  ) for seq in encoding['input_ids']]

             # fmt: off
             __snake_case :   Optional[Any]													=    {
                 'input_ids': [
                     [1, 2118, 11126, 565, 35, 83, 25191, 163, 18854, 13, 12156, 12, 16101, 25376, 13807, 9, 22205, 27893, 1635, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 2118, 11126, 565, 24536, 80, 43797, 4878, 7373, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 133, 78, 65, 16, 10, 3724, 1538, 33183, 11303, 43797, 1938, 4, 870, 24165, 29105, 5, 739, 32644, 33183, 11303, 36173, 88, 80, 650, 7821, 45940, 6, 52, 2559, 5, 1836, 9, 5, 7397, 13171, 31, 5, 1836, 9, 32644, 33183, 11303, 4, 2]
                 ],
                 'token_type_ids': [
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
                 ],
                 'attention_mask': [
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
                     [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
                 ]
             }
             # fmt: on

             __snake_case :   int													=    [
                 'ALBERT: A Lite BERT for Self-supervised Learning of Language Representations',
                 'ALBERT incorporates two parameter reduction techniques',
                 'The first one is a factorized embedding parameterization. By decomposing the large vocabulary'
                 ' embedding matrix into two small matrices, we separate the size of the hidden layers from the size of'
                 ' vocabulary embedding.',
             ]

             self.assertDictEqual(encoding.data      ,     __a  )

             for expected, decoded in zip(__a      ,     __a  ):
                   self.assertEqual(__a      ,     __a  )



