
from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
      from transformers import (
          TFAutoModel,
          TFAutoModelForCausalLM,
          TFAutoModelForMaskedLM,
          TFAutoModelForPreTraining,
          TFAutoModelForQuestionAnswering,
          TFAutoModelForSeqaSeqLM,
          TFAutoModelForSequenceClassification,
          TFAutoModelForTableQuestionAnswering,
          TFAutoModelForTokenClassification,
          TFAutoModelWithLMHead,
          TFBertForMaskedLM,
          TFBertForPreTraining,
          TFBertForQuestionAnswering,
          TFBertForSequenceClassification,
          TFBertModel,
          TFFunnelBaseModel,
          TFFunnelModel,
          TFGPTaLMHeadModel,
          TFRobertaForMaskedLM,
          TFTaForConditionalGeneration,
          TFTapasForQuestionAnswering,
      )
      from transformers.models.auto.modeling_tf_auto import (
          TF_MODEL_FOR_CAUSAL_LM_MAPPING,
          TF_MODEL_FOR_MASKED_LM_MAPPING,
          TF_MODEL_FOR_PRETRAINING_MAPPING,
          TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
          TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
          TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
          TF_MODEL_MAPPING,
      )
      from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
      from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
      from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
      from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST



class 						UpperCamelCase__    (			SCREAMING_SNAKE_CASE_       ):
   _SCREAMING_SNAKE_CASE :				Any      			= "new-model"


if is_tf_available():



      class 						UpperCamelCase__    (			SCREAMING_SNAKE_CASE_       ):
         _SCREAMING_SNAKE_CASE :				Dict      			= NewModelConfig



@require_tf
class 						UpperCamelCase__    (			unittest.TestCase       ):
   @slow
   def  lowerCAmelCase     (self					:   str  ):
    __a					:    Optional[int]													=    'bert-base-cased'
    __a					:    List[str]													=    AutoConfig.from_pretrained(__a  )
    self.assertIsNotNone(__a  )
    self.assertIsInstance(__a   ,							__a  )

    __a					:    Dict													=    TFAutoModel.from_pretrained(__a  )
    self.assertIsNotNone(__a  )
    self.assertIsInstance(__a   ,							__a  )
   @slow
   def  lowerCAmelCase     (self					:   Union[str, Any]  ):
    __a					:    Dict													=    'bert-base-cased'
    __a					:    int													=    AutoConfig.from_pretrained(__a  )
    self.assertIsNotNone(__a  )
    self.assertIsInstance(__a   ,							__a  )

    __a					:    str													=    TFAutoModelForPreTraining.from_pretrained(__a  )
    self.assertIsNotNone(__a  )
    self.assertIsInstance(__a   ,							__a  )
   @slow
   def  lowerCAmelCase     (self					:   int  ):
    for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     __a					:    Dict													=    AutoConfig.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )

     __a					:    Optional[int]													=    TFAutoModelForCausalLM.from_pretrained(__a  )
     __a					:    Optional[int]													=    TFAutoModelForCausalLM.from_pretrained(__a   ,							output_loading_info=__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )
   @slow
   def  lowerCAmelCase     (self					:   List[Any]  ):
    for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     __a					:    Optional[Any]													=    AutoConfig.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )

     __a					:    Optional[int]													=    TFAutoModelWithLMHead.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )
   @slow
   def  lowerCAmelCase     (self					:   Union[str, Any]  ):
    for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     __a					:    Union[str, Any]													=    AutoConfig.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )

     __a					:    int													=    TFAutoModelForMaskedLM.from_pretrained(__a  )
     __a					:    Optional[int]													=    TFAutoModelForMaskedLM.from_pretrained(__a   ,							output_loading_info=__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )
   @slow
   def  lowerCAmelCase     (self					:   List[str]  ):
    for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
     __a					:    Optional[int]													=    AutoConfig.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )

     __a					:    Optional[int]													=    TFAutoModelForSeqaSeqLM.from_pretrained(__a  )
     __a					:    Tuple													=    TFAutoModelForSeqaSeqLM.from_pretrained(__a   ,							output_loading_info=__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )
   @slow
   def  lowerCAmelCase     (self					:   Tuple  ):
    # for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
    for model_name in ["bert-base-uncased"]:
     __a					:    int													=    AutoConfig.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )

     __a					:    List[Any]													=    TFAutoModelForSequenceClassification.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )
   @slow
   def  lowerCAmelCase     (self					:   Union[str, Any]  ):
    # for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
    for model_name in ["bert-base-uncased"]:
     __a					:    List[Any]													=    AutoConfig.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )

     __a					:    Tuple													=    TFAutoModelForQuestionAnswering.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )
   @slow
   @require_tensorflow_probability
   def  lowerCAmelCase     (self					:   Optional[Any]  ):
    for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
     __a					:    List[Any]													=    AutoConfig.from_pretrained(__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )

     __a					:    Tuple													=    TFAutoModelForTableQuestionAnswering.from_pretrained(__a  )
     __a					:    Any													=    TFAutoModelForTableQuestionAnswering.from_pretrained(
         __a   ,							output_loading_info=__a  )
     self.assertIsNotNone(__a  )
     self.assertIsInstance(__a   ,							__a  )
   def  lowerCAmelCase     (self					:   Tuple  ):
    __a					:    str													=    TFAutoModelWithLMHead.from_pretrained(__a  )
    self.assertIsInstance(__a   ,							__a  )
    self.assertEqual(model.num_parameters()   ,							1_4_4_1_0  )
    self.assertEqual(model.num_parameters(only_trainable=__a  )   ,							1_4_4_1_0  )
   def  lowerCAmelCase     (self					:   List[str]  ):
    __a					:    Optional[Any]													=    TFAutoModelWithLMHead.from_pretrained(__a  )
    self.assertIsInstance(__a   ,							__a  )
    self.assertEqual(model.num_parameters()   ,							1_4_4_1_0  )
    self.assertEqual(model.num_parameters(only_trainable=__a  )   ,							1_4_4_1_0  )
   def  lowerCAmelCase     (self					:   int  ):
    __a					:    Optional[int]													=    TFAutoModel.from_pretrained('''sgugger/funnel-random-tiny'''  )
    self.assertIsInstance(__a   ,							__a  )

    __a					:    Optional[Any]													=    copy.deepcopy(model.config  )
    __a					:    Any													=    ['FunnelBaseModel']
    __a					:    Any													=    TFAutoModel.from_config(__a  )
    self.assertIsInstance(__a   ,							__a  )

    with tempfile.TemporaryDirectory() as tmp_dir:
     model.save_pretrained(__a  )
     __a					:    Optional[int]													=    TFAutoModel.from_pretrained(__a  )
     self.assertIsInstance(__a   ,							__a  )
   def  lowerCAmelCase     (self					:   List[Any]  ):
    try:
     AutoConfig.register('''new-model'''   ,							__a  )

     __a					:    Tuple													=    [
         TFAutoModel,
         TFAutoModelForCausalLM,
         TFAutoModelForMaskedLM,
         TFAutoModelForPreTraining,
         TFAutoModelForQuestionAnswering,
         TFAutoModelForSequenceClassification,
         TFAutoModelForTokenClassification,
     ]

     for auto_class in auto_classes:
      with self.subTest(auto_class.__name__  ):
       # Wrong config class will raise an error
       with self.assertRaises(__a  ):
        auto_class.register(__a   ,							__a  )
       auto_class.register(__a   ,							__a  )
       # Trying to register something existing in the Transformers library will raise an error
       with self.assertRaises(__a  ):
        auto_class.register(__a   ,							__a  )

       # Now that the config is registered, it can be used as any other config with the auto-API
       __a					:    Union[str, Any]													=    BertModelTester(self  ).get_config()
       __a					:    Dict													=    NewModelConfig(**tiny_config.to_dict()  )
       __a					:    Union[str, Any]													=    auto_class.from_config(__a  )
       self.assertIsInstance(__a   ,							__a  )

       with tempfile.TemporaryDirectory() as tmp_dir:
        model.save_pretrained(__a  )
        __a					:    Any													=    auto_class.from_pretrained(__a  )
        self.assertIsInstance(__a   ,							__a  )

    finally:
     if "new-model" in CONFIG_MAPPING._extra_content:
      del CONFIG_MAPPING._extra_content["new-model"]
     for mapping in (
         TF_MODEL_MAPPING,
         TF_MODEL_FOR_PRETRAINING_MAPPING,
         TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
         TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
         TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
         TF_MODEL_FOR_CAUSAL_LM_MAPPING,
         TF_MODEL_FOR_MASKED_LM_MAPPING,
     ):
      if NewModelConfig in mapping._extra_content:
       del mapping._extra_content[NewModelConfig]
   def  lowerCAmelCase     (self					:   Any  ):
    with self.assertRaisesRegex(
        __a   ,							'''bert-base is not a local folder and is not a valid model identifier'''  ):
     __a					:    List[Any]													=    TFAutoModel.from_pretrained('''bert-base'''  )
   def  lowerCAmelCase     (self					:   List[Any]  ):
    with self.assertRaisesRegex(
        __a   ,							r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''  ):
     __a					:    int													=    TFAutoModel.from_pretrained(__a   ,							revision='''aaaaaa'''  )
   def  lowerCAmelCase     (self					:   int  ):
    with self.assertRaisesRegex(
        __a   ,							'''hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'''   ,							):
     __a					:    int													=    TFAutoModel.from_pretrained('''hf-internal-testing/config-no-model'''  )
   def  lowerCAmelCase     (self					:   Union[str, Any]  ):
    with self.assertRaisesRegex(__a   ,							'''Use `from_pt=True` to load this model'''  ):
     __a					:    Any													=    TFAutoModel.from_pretrained('''hf-internal-testing/tiny-bert-pt-only'''  )







   def  lowerCAmelCase     (self					:   Dict  ):
    __a					:    List[str]													=    TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''  )
    with RequestCounter() as counter:
     __a					:    List[str]													=    TFAutoModel.from_pretrained('''hf-internal-testing/tiny-random-bert'''  )
     self.assertEqual(counter.get_request_count   ,							0  )
     self.assertEqual(counter.head_request_count   ,							1  )
     self.assertEqual(counter.other_request_count   ,							0  )

    # With a sharded checkpoint
    __a					:    Optional[int]													=    TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''  )
    with RequestCounter() as counter:
     __a					:    int													=    TFAutoModel.from_pretrained('''ArthurZ/tiny-random-bert-sharded'''  )
     self.assertEqual(counter.get_request_count   ,							0  )
     self.assertEqual(counter.head_request_count   ,							1  )
     self.assertEqual(counter.other_request_count   ,							0  )





'''simple docstring'''





import json
import os
import unittest

from transformers.models.blenderbot_small.tokenization_blenderbot_small import (
    VOCAB_FILES_NAMES,
    BlenderbotSmallTokenizer,
)

from ...test_tokenization_common import TokenizerTesterMixin



class  snake_case__						(    SCREAMING_SNAKE_CASE_  ,     unittest.TestCase				):
 A__				= BlenderbotSmallTokenizer
 A__				= False
 def A_   (   self						:							Any  )      ->   str:







       '''simple docstring'''
       super().setUp()

       __snake_case :   Any													=    ['__start__', 'adapt', 'act', 'ap@@', 'te', '__end__', '__unk__']
       __snake_case :   str													=    dict(zip(__a      ,     range(len(__a  )  )  )  )

       __snake_case :   List[Any]													=    ['#version: 0.2', 'a p', 't e</w>', 'ap t</w>', 'a d', 'ad apt</w>', 'a c', 'ac t</w>', '']
       __snake_case :   Tuple													=    {'unk_token': '__unk__', 'bos_token': '__start__', 'eos_token': '__end__'}

       __snake_case :   str													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['vocab_file']  )
       __snake_case :   Union[str, Any]													=    os.path.join(self.tmpdirname      ,     VOCAB_FILES_NAMES['merges_file']  )
       with open(self.vocab_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write(json.dumps(__a  ) + '\n'  )
       with open(self.merges_file      ,     'w'      ,     encoding='utf-8'  ) as fp:
             fp.write('\n'.join(__a  )  )
 def A_   (   self						:							Tuple      ,     **__a						:							Optional[int]  )      ->   List[str]:







       '''simple docstring'''
       kwargs.update(self.special_tokens_map  )
       return BlenderbotSmallTokenizer.from_pretrained(self.tmpdirname      ,     **__a  )
 def A_   (   self						:							int      ,     __a						:							int  )      ->   List[Any]:







       '''simple docstring'''
       __snake_case :   List[str]													=    'adapt act apte'
       __snake_case :   List[str]													=    'adapt act apte'
       return input_text, output_text
 def A_   (   self						:							Dict  )      ->   str:







       '''simple docstring'''
       __snake_case :   Dict													=    BlenderbotSmallTokenizer(self.vocab_file      ,     self.merges_file      ,     **self.special_tokens_map  )
       __snake_case :   List[str]													=    'adapt act apte'
       __snake_case :   List[Any]													=    ['adapt', 'act', 'ap@@', 'te']
       __snake_case :   Optional[int]													=    tokenizer.tokenize(__a  )
       self.assertListEqual(__a      ,     __a  )

       __snake_case :   List[str]													=    [tokenizer.bos_token] + tokens + [tokenizer.eos_token]

       __snake_case :   List[str]													=    [0, 1, 2, 3, 4, 5]
       self.assertListEqual(tokenizer.convert_tokens_to_ids(__a  )      ,     __a  )
 def A_   (   self						:							int  )      ->   str:







       '''simple docstring'''
       __snake_case :   int													=    BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'  )
       assert tok('sam'  ).input_ids == [1384]
       __snake_case :   List[str]													=    'I am a small frog.'
       __snake_case :   List[Any]													=    tok([src_text]      ,     padding=__a      ,     truncation=__a  )['input_ids']
       __snake_case :   int													=    tok.batch_decode(__a      ,     skip_special_tokens=__a      ,     clean_up_tokenization_spaces=__a  )[0]
       assert src_text != decoded  # I wish it did!
       assert decoded == "i am a small frog ."





 def A_   (   self						:							List[str]  )      ->   Union[str, Any]:







       '''simple docstring'''
       __snake_case :   Tuple													=    BlenderbotSmallTokenizer.from_pretrained('facebook/blenderbot-90M'  )
       __snake_case :   Optional[Any]													=    'I am a small frog .'
       __snake_case :   List[str]													=    '.'
       __snake_case :   List[str]													=    tok(__a  )['input_ids']
       __snake_case :   str													=    tok(__a  )['input_ids']

       assert encoded[-1] == encoded_dot[0]



