




from __future__ import annotations

import unittest

import numpy as np

from transformers import BlipTextConfig
from transformers.testing_utils import require_tf, slow
from transformers.utils import is_tf_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask


if is_tf_available():
   import tensorflow as tf

   from transformers import TFBlipTextModel
   from transformers.models.blip.modeling_tf_blip import TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST





class 			lowerCamelCase_				:


 '''simple docstring'''







 def __init__(				self    :						Tuple							,							_lowerCAmelCase    :						Tuple							,							_lowerCAmelCase    :						Optional[Any]=12							,							_lowerCAmelCase    :						Dict=7							,							_lowerCAmelCase    :						Optional[Any]=True							,							_lowerCAmelCase    :						Dict=True							,							_lowerCAmelCase    :						Optional[int]=True							,							_lowerCAmelCase    :						str=99							,							_lowerCAmelCase    :						List[str]=32							,							_lowerCAmelCase    :						int=32							,							_lowerCAmelCase    :						List[Any]=2							,							_lowerCAmelCase    :						List[Any]=4							,							_lowerCAmelCase    :						Optional[Any]=37							,							_lowerCAmelCase    :						int=0.1							,							_lowerCAmelCase    :						List[Any]=0.1							,							_lowerCAmelCase    :						List[Any]=512							,							_lowerCAmelCase    :						List[Any]=0.02							,							_lowerCAmelCase    :						List[Any]=0							,							_lowerCAmelCase    :						Optional[Any]=None							,							):
       SCREAMING_SNAKE_CASE_      		=     parent
       SCREAMING_SNAKE_CASE_      		=     batch_size
       SCREAMING_SNAKE_CASE_      		=     seq_length
       SCREAMING_SNAKE_CASE_      		=     is_training
       SCREAMING_SNAKE_CASE_      		=     use_input_mask
       SCREAMING_SNAKE_CASE_      		=     use_labels
       SCREAMING_SNAKE_CASE_      		=     vocab_size
       SCREAMING_SNAKE_CASE_      		=     hidden_size
       SCREAMING_SNAKE_CASE_      		=     projection_dim
       SCREAMING_SNAKE_CASE_      		=     num_hidden_layers
       SCREAMING_SNAKE_CASE_      		=     num_attention_heads
       SCREAMING_SNAKE_CASE_      		=     intermediate_size
       SCREAMING_SNAKE_CASE_      		=     dropout
       SCREAMING_SNAKE_CASE_      		=     attention_dropout
       SCREAMING_SNAKE_CASE_      		=     max_position_embeddings
       SCREAMING_SNAKE_CASE_      		=     initializer_range
       SCREAMING_SNAKE_CASE_      		=     scope
       SCREAMING_SNAKE_CASE_      		=     bos_token_id



 def  lowerCAmelCase_    (				self    :						Dict		):
       SCREAMING_SNAKE_CASE_      		=     ids_tensor([self.batch_size, self.seq_length]							,							self.vocab_size		)

       SCREAMING_SNAKE_CASE_      		=     None
       if self.use_input_mask:
             SCREAMING_SNAKE_CASE_      		=     random_attention_mask([self.batch_size, self.seq_length]		)

       if input_mask is not None:
             SCREAMING_SNAKE_CASE_      		=     input_mask.numpy()
             SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     input_mask.shape
             SCREAMING_SNAKE_CASE_      		=     np.random.randint(1							,							seq_length - 1							,							size=(batch_size,)		)
             for batch_idx, start_index in enumerate(_lowerCAmelCase		):
                   SCREAMING_SNAKE_CASE_      		=     1
                   SCREAMING_SNAKE_CASE_      		=     0

       SCREAMING_SNAKE_CASE_      		=     self.get_config()

       return config, input_ids, tf.convert_to_tensor(_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						str		):
       return BlipTextConfig(
           vocab_size=self.vocab_size							,							hidden_size=self.hidden_size							,							projection_dim=self.projection_dim							,							num_hidden_layers=self.num_hidden_layers							,							num_attention_heads=self.num_attention_heads							,							intermediate_size=self.intermediate_size							,							dropout=self.dropout							,							attention_dropout=self.attention_dropout							,							max_position_embeddings=self.max_position_embeddings							,							initializer_range=self.initializer_range							,							bos_token_id=self.bos_token_id							,							)



 def  lowerCAmelCase_    (				self    :						Union[str, Any]							,							_lowerCAmelCase    :						Any							,							_lowerCAmelCase    :						int							,							_lowerCAmelCase    :						Any		):
       SCREAMING_SNAKE_CASE_      		=     TFBlipTextModel(config=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     model(_lowerCAmelCase							,							attention_mask=_lowerCAmelCase							,							training=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     model(_lowerCAmelCase							,							training=_lowerCAmelCase		)
       self.parent.assertEqual(result.last_hidden_state.shape							,							(self.batch_size, self.seq_length, self.hidden_size)		)
       self.parent.assertEqual(result.pooler_output.shape							,							(self.batch_size, self.hidden_size)		)




 def  lowerCAmelCase_    (				self    :						Dict		):
       SCREAMING_SNAKE_CASE_      		=     self.prepare_config_and_inputs()
       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     config_and_inputs
       SCREAMING_SNAKE_CASE_      		=     {'input_ids': input_ids, 'attention_mask': input_mask}
       return config, inputs_dict





@require_tf
class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE			,  unittest.TestCase   ):


 '''simple docstring'''







 lowercase_							       =       (TFBlipTextModel,) if is_tf_available() else ()
 lowercase_							       =       False
 lowercase_							       =       False
 lowercase_							       =       False



 def  lowerCAmelCase_    (				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     BlipTextModelTester(self		)
       SCREAMING_SNAKE_CASE_      		=     ConfigTester(self							,							config_class=_lowerCAmelCase							,							hidden_size=37		)



 def  lowerCAmelCase_    (				self    :						Union[str, Any]		):
       self.config_tester.run_common_tests()



 def  lowerCAmelCase_    (				self    :						int		):
       SCREAMING_SNAKE_CASE_      		=     self.model_tester.prepare_config_and_inputs()
       self.model_tester.create_and_check_model(*_lowerCAmelCase		)



 def  lowerCAmelCase_    (				self    :						int		):
       pass



 def  lowerCAmelCase_    (				self    :						List[str]		):
       pass



 @unittest.skip(reason='Blip does not use inputs_embeds'		)
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       pass



 @unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'		)
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       pass



 @unittest.skip(reason='BlipTextModel has no base class and is not available in MODEL_MAPPING'		)
 def  lowerCAmelCase_    (				self    :						str		):
       pass



 @slow
 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       for model_name in TF_BLIP_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
             SCREAMING_SNAKE_CASE_      		=     TFBlipTextModel.from_pretrained(_lowerCAmelCase		)
             self.assertIsNotNone(_lowerCAmelCase		)




 def  lowerCAmelCase_    (				self    :						List[Any]							,							_lowerCAmelCase    :						int=True		):
       super().test_pt_tf_model_equivalence(allow_missing_keys=_lowerCAmelCase		)






import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast





class UpperCAmelCase_				( unittest.TestCase ):


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       0


  @slow
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )

       for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(GPTaTokenizer, GPTaTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(RobertaTokenizer, RobertaTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,2_0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoConfig.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,__snake_case )
       # Check that tokenizer_type ≠ model_type
       A_          =       AutoTokenizer.from_pretrained(__snake_case ,config=__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' )
            self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with pytest.raises(__snake_case ):
            AutoTokenizer.from_pretrained('''./''' ,tokenizer_type='''xxx''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            A_          =       tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''' )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )

            if isinstance(__snake_case ,__snake_case ):
                 self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,__snake_case )
            else:
                 self.assertEqual(tokenizer.do_lower_case ,__snake_case )

            self.assertEqual(tokenizer.model_max_length ,5_1_2 )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            with self.assertRaisesRegex(
                __snake_case ,'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier''' ,):
                 A_          =       tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       TOKENIZER_MAPPING.values()
       A_          =       []

       for slow_tok, fast_tok in tokenizers:
            if slow_tok is not None:
                 tokenizer_names.append(slow_tok.__name__ )

            if fast_tok is not None:
                 tokenizer_names.append(fast_tok.__name__ )

       for tokenizer_name in tokenizer_names:
            # must find the right class
            tokenizer_class_from_name(__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ,use_fast=__snake_case ) ,__snake_case )
       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ) ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''distilbert-base-uncased''' ,do_lower_case=__snake_case )
       A_          =       '''Hello, world. How are you?'''
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )

       A_          =       AutoTokenizer.from_pretrained('''microsoft/mpnet-base''' ,do_lower_case=__snake_case )
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''' )
       self.assertEqual(type(__snake_case ) ,__snake_case )
       self.assertEqual(tokenizer.model_max_length ,5_1_2 )
       self.assertEqual(tokenizer.vocab_size ,3_0_0_0_0 )
       self.assertEqual(tokenizer.unk_token ,'''[UNK]''' )
       self.assertEqual(tokenizer.padding_side ,'''right''' )
       self.assertEqual(tokenizer.truncation_side ,'''right''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case )

       self.assertIsInstance(__snake_case ,tokenizer.__class__ )
       self.assertEqual(tokenizera.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''ctrl''' )
       # There is no fast CTRL so this always gives us a slow tokenizer.
       self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       get_tokenizer_config('''bert-base-cased''' )
       A_          =       config.pop('''_commit_hash''' ,__snake_case )
       # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
       self.assertEqual(__snake_case ,{'''do_lower_case''': False} )

       # This model does not have a tokenizer_config so we get back an empty dict.
       A_          =       get_tokenizer_config(__snake_case )
       self.assertDictEqual(__snake_case ,{} )

       # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       get_tokenizer_config(__snake_case )

       # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
       self.assertEqual(config['''tokenizer_class'''] ,'''BertTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )

            A_          =       CustomTokenizer.from_pretrained(__snake_case )
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            # Can register in two steps
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, None) )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            del TOKENIZER_MAPPING._extra_content[CustomConfig]
            # Can register in one step
            AutoTokenizer.register(
                __snake_case ,slow_tokenizer_class=__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )

            # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
            # and that model does not have a tokenizer.json
            with tempfile.TemporaryDirectory() as tmp_dir:
                 A_          =       BertTokenizerFast.from_pretrained(__snake_case )
                 bert_tokenizer.save_pretrained(__snake_case )
                 A_          =       CustomTokenizerFast.from_pretrained(__snake_case )

            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,use_fast=__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )

       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case )
       self.assertTrue(reloaded_tokenizer.special_attribute_present )

       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            # Test tokenizer can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )
                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(reloaded_tokenizer.special_attribute_present )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[int]							=   False





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[Any]							=   NewTokenizer
         __lowerCAmelCase   :						Union[str, Any]							=   False

       try:
            AutoConfig.register('''custom''' ,__snake_case )
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            # If remote code is not set, the default is to use local
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote code is disabled, we load the local one.
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote is enabled, we load from the Hub
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertTrue(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(tokenizer.special_attribute_present )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(
           '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,'''bert-base is not a local folder and is not a valid model identifier''' ):
            A_          =       AutoTokenizer.from_pretrained('''bert-base''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,revision='''aaaaaa''' )




  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
       with RequestCounter() as counter:
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
            self.assertEqual(counter.get_request_count ,0 )
            self.assertEqual(counter.head_request_count ,1 )
            self.assertEqual(counter.other_request_count ,0 )



