






"""simple docstring"""
from __future__ import annotations

import copy
import tempfile
import unittest

from transformers import CONFIG_MAPPING, AutoConfig, BertConfig, GPTaConfig, TaConfig, TapasConfig, is_tf_available
from transformers.testing_utils import (
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tensorflow_probability,
    require_tf,
    slow,
)

from ..bert.test_modeling_bert import BertModelTester


if is_tf_available():
 from transformers import (
     TFAutoModel,
     TFAutoModelForCausalLM,
     TFAutoModelForMaskedLM,
     TFAutoModelForPreTraining,
     TFAutoModelForQuestionAnswering,
     TFAutoModelForSeqaSeqLM,
     TFAutoModelForSequenceClassification,
     TFAutoModelForTableQuestionAnswering,
     TFAutoModelForTokenClassification,
     TFAutoModelWithLMHead,
     TFBertForMaskedLM,
     TFBertForPreTraining,
     TFBertForQuestionAnswering,
     TFBertForSequenceClassification,
     TFBertModel,
     TFFunnelBaseModel,
     TFFunnelModel,
     TFGPTaLMHeadModel,
     TFRobertaForMaskedLM,
     TFTaForConditionalGeneration,
     TFTapasForQuestionAnswering,
 )
 from transformers.models.auto.modeling_tf_auto import (
     TF_MODEL_FOR_CAUSAL_LM_MAPPING,
     TF_MODEL_FOR_MASKED_LM_MAPPING,
     TF_MODEL_FOR_PRETRAINING_MAPPING,
     TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
     TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
     TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
     TF_MODEL_MAPPING,
 )
 from transformers.models.bert.modeling_tf_bert import TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.gpta.modeling_tf_gpta import TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.ta.modeling_tf_ta import TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST
 from transformers.models.tapas.modeling_tf_tapas import TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST



class        UpperCamelCase__ (				a_):







    """simple docstring"""
    __UpperCAmelCase           = """new-model"""


if is_tf_available():



 class        UpperCamelCase__ (				a_):







     """simple docstring"""
     __UpperCAmelCase           = NewModelConfig



@require_tf
class        UpperCamelCase__ (				unittest.TestCase):







    """simple docstring"""







    @slow
    def 				a__			(		self :	int ):





           '''simple docstring'''

           __magic_name__      				=    'bert-base-cased'
           __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
           self.assertIsNotNone(UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

           __magic_name__      				=    TFAutoModel.from_pretrained(UpperCamelCase_ )
           self.assertIsNotNone(UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	str ):





           '''simple docstring'''

           __magic_name__      				=    'bert-base-cased'
           __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
           self.assertIsNotNone(UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

           __magic_name__      				=    TFAutoModelForPreTraining.from_pretrained(UpperCamelCase_ )
           self.assertIsNotNone(UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	Any ):





           '''simple docstring'''

           for model_name in TF_GPT2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                  __magic_name__      				=    TFAutoModelForCausalLM.from_pretrained(UpperCamelCase_ )
                  __magic_name__				,  __magic_name__      				=    TFAutoModelForCausalLM.from_pretrained(UpperCamelCase_				,    output_loading_info=UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	Optional[Any] ):





           '''simple docstring'''

           for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                  __magic_name__      				=    TFAutoModelWithLMHead.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	Dict ):





           '''simple docstring'''

           for model_name in TF_BERT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                  __magic_name__      				=    TFAutoModelForMaskedLM.from_pretrained(UpperCamelCase_ )
                  __magic_name__				,  __magic_name__      				=    TFAutoModelForMaskedLM.from_pretrained(UpperCamelCase_				,    output_loading_info=UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	Any ):





           '''simple docstring'''

           for model_name in TF_T5_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                  __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                  __magic_name__      				=    TFAutoModelForSeqaSeqLM.from_pretrained(UpperCamelCase_ )
                  __magic_name__				,  __magic_name__      				=    TFAutoModelForSeqaSeqLM.from_pretrained(UpperCamelCase_				,    output_loading_info=UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	str ):





           '''simple docstring'''

           for model_name in ["bert-base-uncased"]:
                  __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                  __magic_name__      				=    TFAutoModelForSequenceClassification.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    def 				a__			(		self :	Any ):





           '''simple docstring'''

           for model_name in ["bert-base-uncased"]:
                  __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                  __magic_name__      				=    TFAutoModelForQuestionAnswering.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    @slow
    @require_tensorflow_probability
    def 				a__			(		self :	Dict ):





           '''simple docstring'''

           for model_name in TF_TAPAS_PRETRAINED_MODEL_ARCHIVE_LIST[5:6]:
                  __magic_name__      				=    AutoConfig.from_pretrained(UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                  __magic_name__      				=    TFAutoModelForTableQuestionAnswering.from_pretrained(UpperCamelCase_ )
                  __magic_name__				,  __magic_name__      				=    TFAutoModelForTableQuestionAnswering.from_pretrained(
                      UpperCamelCase_				,    output_loading_info=UpperCamelCase_ )
                  self.assertIsNotNone(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    def 				a__			(		self :	str ):





           '''simple docstring'''

           __magic_name__      				=    TFAutoModelWithLMHead.from_pretrained(UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )
           self.assertEqual(model.num_parameters()				,    1_4_4_1_0 )
           self.assertEqual(model.num_parameters(only_trainable=UpperCamelCase_ )				,    1_4_4_1_0 )







    def 				a__			(		self :	List[str] ):





           '''simple docstring'''

           __magic_name__      				=    TFAutoModelWithLMHead.from_pretrained(UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )
           self.assertEqual(model.num_parameters()				,    1_4_4_1_0 )
           self.assertEqual(model.num_parameters(only_trainable=UpperCamelCase_ )				,    1_4_4_1_0 )







    def 				a__			(		self :	List[str] ):





           '''simple docstring'''

           __magic_name__      				=    TFAutoModel.from_pretrained('sgugger/funnel-random-tiny' )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

           __magic_name__      				=    copy.deepcopy(model.config )
           __magic_name__      				=    ['FunnelBaseModel']
           __magic_name__      				=    TFAutoModel.from_config(UpperCamelCase_ )
           self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

           with tempfile.TemporaryDirectory() as tmp_dir:
                  model.save_pretrained(UpperCamelCase_ )
                  __magic_name__      				=    TFAutoModel.from_pretrained(UpperCamelCase_ )
                  self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )







    def 				a__			(		self :	int ):





           '''simple docstring'''

           try:
                  AutoConfig.register('new-model'				,    UpperCamelCase_ )

                  __magic_name__      				=    [
                      TFAutoModel,
                      TFAutoModelForCausalLM,
                      TFAutoModelForMaskedLM,
                      TFAutoModelForPreTraining,
                      TFAutoModelForQuestionAnswering,
                      TFAutoModelForSequenceClassification,
                      TFAutoModelForTokenClassification,
                  ]

                  for auto_class in auto_classes:
                         with self.subTest(auto_class.__name__ ):
                                # Wrong config class will raise an error
                                with self.assertRaises(UpperCamelCase_ ):
                                       auto_class.register(UpperCamelCase_				,    UpperCamelCase_ )
                                auto_class.register(UpperCamelCase_				,    UpperCamelCase_ )
                                # Trying to register something existing in the Transformers library will raise an error
                                with self.assertRaises(UpperCamelCase_ ):
                                       auto_class.register(UpperCamelCase_				,    UpperCamelCase_ )

                                # Now that the config is registered, it can be used as any other config with the auto-API
                                __magic_name__      				=    BertModelTester(self ).get_config()
                                __magic_name__      				=    NewModelConfig(**tiny_config.to_dict() )
                                __magic_name__      				=    auto_class.from_config(UpperCamelCase_ )
                                self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

                                with tempfile.TemporaryDirectory() as tmp_dir:
                                       model.save_pretrained(UpperCamelCase_ )
                                       __magic_name__      				=    auto_class.from_pretrained(UpperCamelCase_ )
                                       self.assertIsInstance(UpperCamelCase_				,    UpperCamelCase_ )

           finally:
                  if "new-model" in CONFIG_MAPPING._extra_content:
                         del CONFIG_MAPPING._extra_content["new-model"]
                  for mapping in (
                      TF_MODEL_MAPPING,
                      TF_MODEL_FOR_PRETRAINING_MAPPING,
                      TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
                      TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
                      TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
                      TF_MODEL_FOR_CAUSAL_LM_MAPPING,
                      TF_MODEL_FOR_MASKED_LM_MAPPING,
                  ):
                         if NewModelConfig in mapping._extra_content:
                                del mapping._extra_content[NewModelConfig]







    def 				a__			(		self :	Any ):





           '''simple docstring'''

           with self.assertRaisesRegex(
               UpperCamelCase_				,    'bert-base is not a local folder and is not a valid model identifier' ):
                  __magic_name__      				=    TFAutoModel.from_pretrained('bert-base' )







    def 				a__			(		self :	Dict ):





           '''simple docstring'''

           with self.assertRaisesRegex(
               UpperCamelCase_				,    r'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)' ):
                  __magic_name__      				=    TFAutoModel.from_pretrained(UpperCamelCase_				,    revision='aaaaaa' )







    def 				a__			(		self :	str ):





           '''simple docstring'''

           with self.assertRaisesRegex(
               UpperCamelCase_				,    'hf-internal-testing/config-no-model does not appear to have a file named pytorch_model.bin'				,    ):
                  __magic_name__      				=    TFAutoModel.from_pretrained('hf-internal-testing/config-no-model' )







    def 				a__			(		self :	str ):





           '''simple docstring'''

           with self.assertRaisesRegex(UpperCamelCase_				,    'Use `from_pt=True` to load this model' ):
                  __magic_name__      				=    TFAutoModel.from_pretrained('hf-internal-testing/tiny-bert-pt-only' )







    def 				a__			(		self :	str ):





           '''simple docstring'''

           __magic_name__      				=    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert' )
           with RequestCounter() as counter:
                  __magic_name__      				=    TFAutoModel.from_pretrained('hf-internal-testing/tiny-random-bert' )
                  self.assertEqual(counter.get_request_count				,    0 )
                  self.assertEqual(counter.head_request_count				,    1 )
                  self.assertEqual(counter.other_request_count				,    0 )

           # With a sharded checkpoint
           __magic_name__      				=    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded' )
           with RequestCounter() as counter:
                  __magic_name__      				=    TFAutoModel.from_pretrained('ArthurZ/tiny-random-bert-sharded' )
                  self.assertEqual(counter.get_request_count				,    0 )
                  self.assertEqual(counter.head_request_count				,    1 )
                  self.assertEqual(counter.other_request_count				,    0 )







"""simple docstring"""
import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_mobilebert import MobileBertTokenizer


snake_case : Optional[Any]     		=		logging.get_logger(__name__)

snake_case : str     		=		{"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

snake_case : List[Any]     		=		{
    """vocab_file""": {"""mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/vocab.txt"""},
    """tokenizer_file""": {
        """mobilebert-uncased""": """https://huggingface.co/google/mobilebert-uncased/resolve/main/tokenizer.json"""
    },
}

snake_case : Tuple     		=		{"""mobilebert-uncased""": 5_1_2}


snake_case : List[str]     		=		{}



class        UpperCamelCase__ (				a_):







    """simple docstring"""
    __UpperCAmelCase           = VOCAB_FILES_NAMES
    __UpperCAmelCase           = PRETRAINED_VOCAB_FILES_MAP
    __UpperCAmelCase           = PRETRAINED_INIT_CONFIGURATION
    __UpperCAmelCase           = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __UpperCAmelCase           = MobileBertTokenizer







    def __init__(		self :	Optional[int]				,    UpperCamelCase_ :	Optional[int]=None				,    UpperCamelCase_ :	str=None				,    UpperCamelCase_ :	Tuple=True				,    UpperCamelCase_ :	Any="[UNK]"				,    UpperCamelCase_ :	str="[SEP]"				,    UpperCamelCase_ :	Any="[PAD]"				,    UpperCamelCase_ :	Dict="[CLS]"				,    UpperCamelCase_ :	str="[MASK]"				,    UpperCamelCase_ :	Union[str, Any]=True				,    UpperCamelCase_ :	Dict=None				,    **UpperCamelCase_ :	Dict				,    ):





           '''simple docstring'''

           super().__init__(
               UpperCamelCase_				,    tokenizer_file=UpperCamelCase_				,    do_lower_case=UpperCamelCase_				,    unk_token=UpperCamelCase_				,    sep_token=UpperCamelCase_				,    pad_token=UpperCamelCase_				,    cls_token=UpperCamelCase_				,    mask_token=UpperCamelCase_				,    tokenize_chinese_chars=UpperCamelCase_				,    strip_accents=UpperCamelCase_				,    **UpperCamelCase_				,    )

           __magic_name__      				=    json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get('lowercase'				,    UpperCamelCase_ ) != do_lower_case
               or normalizer_state.get('strip_accents'				,    UpperCamelCase_ ) != strip_accents
               or normalizer_state.get('handle_chinese_chars'				,    UpperCamelCase_ ) != tokenize_chinese_chars
           ):
                  __magic_name__      				=    getattr(UpperCamelCase_				,    normalizer_state.pop('type' ) )
                  __magic_name__      				=    do_lower_case
                  __magic_name__      				=    strip_accents
                  __magic_name__      				=    tokenize_chinese_chars
                  __magic_name__      				=    normalizer_class(**UpperCamelCase_ )

           __magic_name__      				=    do_lower_case







    def 				a__			(		self :	Dict				,    UpperCamelCase_ :	List[str]				,    UpperCamelCase_ :	List[Any]=None ):





           '''simple docstring'''

           __magic_name__      				=    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                  output += token_ids_a + [self.sep_token_id]

           return output







    def 				a__			(		self :	List[Any]				,    UpperCamelCase_ :	List[int]				,    UpperCamelCase_ :	Optional[List[int]] = None ):





           '''simple docstring'''

           __magic_name__      				=    [self.sep_token_id]
           __magic_name__      				=    [self.cls_token_id]
           if token_ids_a is None:
                  return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]







    def 				a__			(		self :	List[str]				,    UpperCamelCase_ :	str				,    UpperCamelCase_ :	Optional[str] = None ):





           '''simple docstring'''

           __magic_name__      				=    self._tokenizer.model.save(UpperCamelCase_				,    name=UpperCamelCase_ )
           return tuple(UpperCamelCase_ )