




'''simple docstring'''







from __future__ import annotations

import unittest

from transformers import XGLMConfig, XGLMTokenizer, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
    import tensorflow as tf

    from transformers.models.xglm.modeling_tf_xglm import (
        TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
        TFXGLMForCausalLM,
        TFXGLMModel,
    )

@require_tf
class 	_snake_case:
 __snake_case:							int						=					XGLMConfig
 __snake_case:							List[Any]						=					{}
 __snake_case:							List[str]						=					'''gelu'''






 def __init__(self       :  str    ,		a       :  Any    ,		a       :  Tuple=14    ,		a       :  Optional[Any]=7    ,		a       :  str=True    ,		a       :  Dict=True    ,		a       :  List[Any]=True    ,		a       :  List[Any]=99    ,		a       :  List[str]=32    ,		a       :  Tuple=2    ,		a       :  Any=4    ,		a       :  List[str]=37    ,		a       :  int="gelu"    ,		a       :  str=0.1    ,		a       :  List[Any]=0.1    ,		a       :  Union[str, Any]=5_12    ,		a       :  int=0.02    ,		)       ->   List[Any]:






    """simple docstring"""


    A__       =     parent
    A__       =     batch_size
    A__       =     seq_length
    A__       =     is_training
    A__       =     use_input_mask
    A__       =     use_labels
    A__       =     vocab_size
    A__       =     d_model
    A__       =     num_hidden_layers
    A__       =     num_attention_heads
    A__       =     ffn_dim
    A__       =     activation_function
    A__       =     activation_dropout
    A__       =     attention_dropout
    A__       =     max_position_embeddings
    A__       =     initializer_range
    A__       =     None
    A__       =     0
    A__       =     2
    A__       =     1






 def 						_UpperCamelCase (self       :  List[str]    )       ->   Optional[int]:






    """simple docstring"""


    return XGLMConfig.from_pretrained('facebook/xglm-564M'    )






 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   Optional[Any]:






    """simple docstring"""


    A__       =     tf.clip_by_value(
        ids_tensor([self.batch_size, self.seq_length]    ,		self.vocab_size    )    ,		clip_value_min=0    ,		clip_value_max=3    )

    A__       =     None
    if self.use_input_mask:
       A__       =     random_attention_mask([self.batch_size, self.seq_length]    )

    A__       =     self.get_config()

    A__       =     floats_tensor([self.num_hidden_layers, self.num_attention_heads]    ,		2    )

    return (
        config,
        input_ids,
        input_mask,
        head_mask,
    )






 def 						_UpperCamelCase (self       :  List[Any]    )       ->   Optional[Any]:






    """simple docstring"""


    return XGLMConfig(
        vocab_size=self.vocab_size    ,		d_model=self.hidden_size    ,		num_layers=self.num_hidden_layers    ,		attention_heads=self.num_attention_heads    ,		ffn_dim=self.ffn_dim    ,		activation_function=self.activation_function    ,		activation_dropout=self.activation_dropout    ,		attention_dropout=self.attention_dropout    ,		max_position_embeddings=self.max_position_embeddings    ,		initializer_range=self.initializer_range    ,		use_cache=a    ,		bos_token_id=self.bos_token_id    ,		eos_token_id=self.eos_token_id    ,		pad_token_id=self.pad_token_id    ,		return_dict=a    ,		)






 def 						_UpperCamelCase (self       :  int    )       ->   str:






    """simple docstring"""


    A__       =     self.prepare_config_and_inputs()

    (
        (
        A__
    )  ,			(
        A__
    )  ,			(
        A__
    )  ,			(
        A__
    )  ,			
    )       =     config_and_inputs

    A__       =     {
        'input_ids': input_ids,
        'head_mask': head_mask,
    }

    return config, inputs_dict




@require_tf
class 	_snake_case(			UpperCAmelCase       , UpperCAmelCase       , unittest.TestCase      ):
 __snake_case:							List[Any]						=					(TFXGLMModel, TFXGLMForCausalLM) if is_tf_available() else ()
 __snake_case:							Optional[Any]						=					(TFXGLMForCausalLM,) if is_tf_available() else ()
 __snake_case:							Optional[int]						=					(
     {'''feature-extraction''': TFXGLMModel, '''text-generation''': TFXGLMForCausalLM} if is_tf_available() else {}
 )
 __snake_case:							Dict						=					False
 __snake_case:							Optional[int]						=					False
 __snake_case:							str						=					False






 def 						_UpperCamelCase (self       :  List[Any]    )       ->   int:






    """simple docstring"""


    A__       =     TFXGLMModelTester(self    )
    A__       =     ConfigTester(self    ,		config_class=a    ,		n_embd=37    )






 def 						_UpperCamelCase (self       :  Dict    )       ->   List[str]:






    """simple docstring"""


    self.config_tester.run_common_tests()






 @slow
 def 						_UpperCamelCase (self       :  Optional[int]    )       ->   Optional[Any]:






    """simple docstring"""


    for model_name in TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
       A__       =     TFXGLMModel.from_pretrained(a    )
       self.assertIsNotNone(a    )






 @unittest.skip(reason='Currently, model embeddings are going to undergo a major refactor.'    )
 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   Tuple:






    """simple docstring"""


    super().test_resize_token_embeddings()




@require_tf
class 	_snake_case(			unittest.TestCase      ):






 @slow
 def 						_UpperCamelCase (self       :  Any    ,		a       :  str=True    )       ->   Optional[int]:






    """simple docstring"""


    A__       =     TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'    )
    A__       =     tf.convert_to_tensor([[2, 2_68, 98_65]]    ,		dtype=tf.intaa    )  # The dog
    # </s> The dog is a very friendly dog. He is very affectionate and loves to play with other
    # fmt: off
    A__       =     [2, 2_68, 98_65, 67, 11, 19_88, 5_72_52, 98_65, 5, 9_84, 67, 19_88, 21_38_38, 16_58, 53, 7_04_46, 33, 66_57, 2_78, 15_81]
    # fmt: on
    A__       =     model.generate(a    ,		do_sample=a    ,		num_beams=1    )
    if verify_outputs:
       self.assertListEqual(output_ids[0].numpy().tolist()    ,		a    )






 @slow
 def 						_UpperCamelCase (self       :  Optional[Any]    )       ->   List[str]:






    """simple docstring"""


    A__       =     XGLMTokenizer.from_pretrained('facebook/xglm-564M'    )
    A__       =     TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'    )

    tf.random.set_seed(0    )
    A__       =     tokenizer('Today is a nice day and'    ,		return_tensors='tf'    )
    A__       =     tokenized.input_ids
    # forces the generation to happen on CPU, to avoid GPU-related quirks (and assure same output regardless of the available devices)
    with tf.device(':/CPU:0'    ):
       A__       =     model.generate(a    ,		do_sample=a    ,		seed=[7, 0]    )
    A__       =     tokenizer.decode(output_ids[0]    ,		skip_special_tokens=a    )

    A__       =     (
        'Today is a nice day and warm evening here over Southern Alberta!! Today when they closed schools due'
    )
    self.assertEqual(a    ,		a    )






 @slow
 def 						_UpperCamelCase (self       :  Any    )       ->   int:






    """simple docstring"""


    A__       =     TFXGLMForCausalLM.from_pretrained('facebook/xglm-564M'    )
    A__       =     XGLMTokenizer.from_pretrained('facebook/xglm-564M'    )

    A__       =     'left'

    # use different length sentences to test batching
    A__       =     [
        'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
        'left-padding, such as in batched generation. The output for the sequence below should be the same '
        'regardless of whether left padding is applied or not. When',
        'Hello, my dog is a little',
    ]

    A__       =     tokenizer(a    ,		return_tensors='tf'    ,		padding=a    )
    A__       =     inputs['input_ids']

    A__       =     model.generate(input_ids=a    ,		attention_mask=inputs['attention_mask']    ,		max_new_tokens=12    )

    A__       =     tokenizer(sentences[0]    ,		return_tensors='tf'    ).input_ids
    A__       =     model.generate(input_ids=a    ,		max_new_tokens=12    )

    A__       =     tokenizer(sentences[1]    ,		return_tensors='tf'    ).input_ids
    A__       =     model.generate(input_ids=a    ,		max_new_tokens=12    )

    A__       =     tokenizer.batch_decode(a    ,		skip_special_tokens=a    )
    A__       =     tokenizer.decode(output_non_padded[0]    ,		skip_special_tokens=a    )
    A__       =     tokenizer.decode(output_padded[0]    ,		skip_special_tokens=a    )

    A__       =     [
        'This is an extremelly long sentence that only exists to test the ability of the model to cope with '
        'left-padding, such as in batched generation. The output for the sequence below should be the same '
        'regardless of whether left padding is applied or not. When left padding is applied, the sequence will be '
        'a single',
        'Hello, my dog is a little bit of a shy one, but he is very friendly',
    ]
    self.assertListEqual(a    ,		a    )
    self.assertListEqual(a    ,		[non_padded_sentence, padded_sentence]    )








'''simple docstring'''







import argparse
import collections
import os
import re
import tempfile

import pandas as pd
from datasets import Dataset
from huggingface_hub import hf_hub_download, upload_folder

from transformers.utils import direct_transformers_import


# All paths are set with the intent you should run this script from the root of the repo with the command
# python utils/update_metadata.py
lowerCAmelCase_     						=				'''src/transformers'''


# This is to make sure the transformers module imported is the one in the repo.
lowerCAmelCase_     						=				direct_transformers_import(TRANSFORMERS_PATH)


# Regexes that match TF/Flax/PT model names.
lowerCAmelCase_     						=				re.compile(R'''TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
lowerCAmelCase_     						=				re.compile(R'''Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')
# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.
lowerCAmelCase_     						=				re.compile(R'''(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)''')


# Fill this with tuples (pipeline_tag, model_mapping, auto_model)
lowerCAmelCase_     						=				[
    ('''pretraining''', '''MODEL_FOR_PRETRAINING_MAPPING_NAMES''', '''AutoModelForPreTraining'''),
    ('''feature-extraction''', '''MODEL_MAPPING_NAMES''', '''AutoModel'''),
    ('''audio-classification''', '''MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForAudioClassification'''),
    ('''text-generation''', '''MODEL_FOR_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForCausalLM'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_CTC_MAPPING_NAMES''', '''AutoModelForCTC'''),
    ('''image-classification''', '''MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForImageClassification'''),
    ('''image-segmentation''', '''MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES''', '''AutoModelForImageSegmentation'''),
    ('''fill-mask''', '''MODEL_FOR_MASKED_LM_MAPPING_NAMES''', '''AutoModelForMaskedLM'''),
    ('''object-detection''', '''MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES''', '''AutoModelForObjectDetection'''),
    (
        '''zero-shot-object-detection''',
        '''MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES''',
        '''AutoModelForZeroShotObjectDetection''',
    ),
    ('''question-answering''', '''MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES''', '''AutoModelForQuestionAnswering'''),
    ('''text2text-generation''', '''MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES''', '''AutoModelForSeq2SeqLM'''),
    ('''text-classification''', '''MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForSequenceClassification'''),
    ('''automatic-speech-recognition''', '''MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES''', '''AutoModelForSpeechSeq2Seq'''),
    (
        '''table-question-answering''',
        '''MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForTableQuestionAnswering''',
    ),
    ('''token-classification''', '''MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForTokenClassification'''),
    ('''multiple-choice''', '''MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES''', '''AutoModelForMultipleChoice'''),
    (
        '''next-sentence-prediction''',
        '''MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES''',
        '''AutoModelForNextSentencePrediction''',
    ),
    (
        '''audio-frame-classification''',
        '''MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForAudioFrameClassification''',
    ),
    ('''audio-xvector''', '''MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES''', '''AutoModelForAudioXVector'''),
    (
        '''document-question-answering''',
        '''MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForDocumentQuestionAnswering''',
    ),
    (
        '''visual-question-answering''',
        '''MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES''',
        '''AutoModelForVisualQuestionAnswering''',
    ),
    ('''image-to-text''', '''MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES''', '''AutoModelForVision2Seq'''),
    (
        '''zero-shot-image-classification''',
        '''MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES''',
        '''AutoModelForZeroShotImageClassification''',
    ),
    ('''depth-estimation''', '''MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES''', '''AutoModelForDepthEstimation'''),
    ('''video-classification''', '''MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES''', '''AutoModelForVideoClassification'''),
    ('''mask-generation''', '''MODEL_FOR_MASK_GENERATION_MAPPING_NAMES''', '''AutoModelForMaskGeneration'''),
]







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   A__       =     re.finditer('.+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)'      ,UpperCAmelCase						)
   return [m.group(0						) for m in matches]







def 						_A      ( ):






   '''simple docstring'''







   A__       =     transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES
   A__       =     {
       config.replace('Config'      ,''						): model_type for model_type, config in config_maping_names.items()
   }

   # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.
   A__       =     collections.defaultdict(UpperCAmelCase						)
   A__       =     collections.defaultdict(UpperCAmelCase						)
   A__       =     collections.defaultdict(UpperCAmelCase						)

   # Let's lookup through all transformers object (once) and find if models are supported by a given backend.
   for attr_name in dir(UpperCAmelCase						):
      A__       =     None
      if _re_tf_models.match(UpperCAmelCase						) is not None:
         A__       =     tf_models
         A__       =     _re_tf_models.match(UpperCAmelCase						).groups()[0]
      elif _re_flax_models.match(UpperCAmelCase						) is not None:
         A__       =     flax_models
         A__       =     _re_flax_models.match(UpperCAmelCase						).groups()[0]
      elif _re_pt_models.match(UpperCAmelCase						) is not None:
         A__       =     pt_models
         A__       =     _re_pt_models.match(UpperCAmelCase						).groups()[0]

      if lookup_dict is not None:
         while len(UpperCAmelCase						) > 0:
            if attr_name in model_prefix_to_model_type:
               A__       =     True
               break
            # Try again after removing the last word in the name
            A__       =     ''.join(camel_case_split(UpperCAmelCase						)[:-1]						)

   A__       =     set(list(pt_models.keys()						) + list(tf_models.keys()						) + list(flax_models.keys()						)						)
   A__       =     list(UpperCAmelCase						)
   all_models.sort()

   A__       =     {'model_type': all_models}
   A__       =     [pt_models[t] for t in all_models]
   A__       =     [tf_models[t] for t in all_models]
   A__       =     [flax_models[t] for t in all_models]

   # Now let's use the auto-mapping names to make sure
   A__       =     {}
   for t in all_models:
      if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:
         A__       =     'AutoProcessor'
      elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:
         A__       =     'AutoTokenizer'
      elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:
         A__       =     'AutoFeatureExtractor'
      else:
         # Default to AutoTokenizer if a model has nothing, for backward compatibility.
         A__       =     'AutoTokenizer'

   A__       =     [processors[t] for t in all_models]

   return pd.DataFrame(UpperCAmelCase						)







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   A__       =     [
       transformers_module.models.auto.modeling_auto,
       transformers_module.models.auto.modeling_tf_auto,
       transformers_module.models.auto.modeling_flax_auto,
   ]
   for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:
      A__       =     [model_mapping, F"""TF_{model_mapping}""", F"""FLAX_{model_mapping}"""]
      A__       =     [auto_class, F"""TF_{auto_class}""", F"""Flax_{auto_class}"""]
      # Loop through all three frameworks
      for module, cls, mapping in zip(UpperCAmelCase      ,UpperCAmelCase      ,UpperCAmelCase						):
         # The type of pipeline may not exist in this framework
         if not hasattr(UpperCAmelCase      ,UpperCAmelCase						):
            continue
         # First extract all model_names
         A__       =     []
         for name in getattr(UpperCAmelCase      ,UpperCAmelCase						).values():
            if isinstance(UpperCAmelCase      ,UpperCAmelCase						):
               model_names.append(UpperCAmelCase						)
            else:
               model_names.extend(list(UpperCAmelCase						)						)

            # Add pipeline tag and auto model class for those models
         table.update({model_name: (pipeline_tag, cls) for model_name in model_names}						)

   return table







def 						_A      ( UpperCAmelCase      ,UpperCAmelCase						):






   '''simple docstring'''







   A__       =     get_frameworks_table()
   A__       =     Dataset.from_pandas(UpperCAmelCase						)

   A__       =     hf_hub_download(
       'huggingface/transformers-metadata'      ,'pipeline_tags.json'      ,repo_type='dataset'      ,token=UpperCAmelCase						)
   A__       =     Dataset.from_json(UpperCAmelCase						)
   A__       =     {
       tags_dataset[i]['model_class']: (tags_dataset[i]['pipeline_tag'], tags_dataset[i]['auto_class'])
       for i in range(len(UpperCAmelCase						)						)
   }
   A__       =     update_pipeline_and_auto_class_table(UpperCAmelCase						)

   # Sort the model classes to avoid some nondeterministic updates to create false update commits.
   A__       =     sorted(table.keys()						)
   A__       =     pd.DataFrame(
       {
           'model_class': model_classes,
           'pipeline_tag': [table[m][0] for m in model_classes],
           'auto_class': [table[m][1] for m in model_classes],
       }						)
   A__       =     Dataset.from_pandas(UpperCAmelCase						)

   with tempfile.TemporaryDirectory() as tmp_dir:
      frameworks_dataset.to_json(os.path.join(UpperCAmelCase      ,'frameworks.json'						)						)
      tags_dataset.to_json(os.path.join(UpperCAmelCase      ,'pipeline_tags.json'						)						)

      if commit_sha is not None:
         A__       =     (
             F"""Update with commit {commit_sha}\n\nSee: """
             F"""https://github.com/huggingface/transformers/commit/{commit_sha}"""
         )
      else:
         A__       =     'Update'

      upload_folder(
          repo_id='huggingface/transformers-metadata'      ,folder_path=UpperCAmelCase      ,repo_type='dataset'      ,token=UpperCAmelCase      ,commit_message=UpperCAmelCase      ,)







def 						_A      ( ):






   '''simple docstring'''







   A__       =     {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}
   A__       =     transformers_module.pipelines.SUPPORTED_TASKS
   A__       =     []
   for key in pipeline_tasks:
      if key not in in_table:
         A__       =     pipeline_tasks[key]['pt']
         if isinstance(UpperCAmelCase      ,(list, tuple)						):
            A__       =     model[0]
         A__       =     model.__name__
         if model not in in_table.values():
            missing.append(UpperCAmelCase						)

   if len(UpperCAmelCase						) > 0:
      A__       =     ', '.join(UpperCAmelCase						)
      raise ValueError(
          'The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside '
          F"""`utils/update_metadata.py`: {msg}. Please add them!"""						)


if __name__ == "__main__":
    lowerCAmelCase_     						=				argparse.ArgumentParser()
    parser.add_argument('''--token''', type=str, help='''The token to use to push to the transformers-metadata dataset.''')
    parser.add_argument('''--commit_sha''', type=str, help='''The sha of the commit going with this update.''')
    parser.add_argument('''--check-only''', action='''store_true''', help='''Activate to just check all pipelines are present.''')
    lowerCAmelCase_     						=				parser.parse_args()

    if args.check_only:
        check_pipeline_tags()
    else:
        update_metadata(args.token, args.commit_sha)


