





import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast

class      lowerCAmelCase							(      unittest.TestCase    ):
 def   A_			(   self						:   int			)    ->			Tuple:
     lowerCamelCase__					:   Optional[int]      				=	0
 @slow
 def   A_			(   self						:   Any			)    ->			int:
     for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
         lowerCamelCase__					:   Optional[int]      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
         self.assertIsNotNone(UpperCAmelCase			)
         self.assertIsInstance(UpperCAmelCase    ,  (BertTokenizer, BertTokenizerFast)			)
         self.assertGreater(len(UpperCAmelCase			)    ,  0			)

     for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
         lowerCamelCase__					:   int      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
         self.assertIsNotNone(UpperCAmelCase			)
         self.assertIsInstance(UpperCAmelCase    ,  (GPTaTokenizer, GPTaTokenizerFast)			)
         self.assertGreater(len(UpperCAmelCase			)    ,  0			)
 def   A_			(   self						:   Optional[Any]			)    ->			int:
     lowerCamelCase__					:   Any      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
     self.assertIsInstance(UpperCAmelCase    ,  (BertTokenizer, BertTokenizerFast)			)
     self.assertEqual(tokenizer.vocab_size    ,  12			)
 def   A_			(   self						:   List[Any]			)    ->			List[str]:
     lowerCamelCase__					:   Dict      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
     self.assertIsInstance(UpperCAmelCase    ,  (RobertaTokenizer, RobertaTokenizerFast)			)
     self.assertEqual(tokenizer.vocab_size    ,  20			)
 def   A_			(   self						:   Optional[int]			)    ->			Any:
     lowerCamelCase__					:   Optional[int]      				=	AutoConfig.from_pretrained(UpperCAmelCase			)
     self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)
     # Check that tokenizer_type ≠ model_type
     lowerCamelCase__					:   Optional[int]      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  config=UpperCAmelCase			)
     self.assertIsInstance(UpperCAmelCase    ,  (BertTokenizer, BertTokenizerFast)			)
     self.assertEqual(tokenizer.vocab_size    ,  12			)
 def   A_			(   self						:   int			)    ->			Any:
     with tempfile.TemporaryDirectory() as tmp_dir:
         shutil.copy('./tests/fixtures/vocab.txt'    ,  os.path.join(UpperCAmelCase    ,  'vocab.txt'			)			)

         lowerCamelCase__					:   int      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  tokenizer_type='bert'    ,  use_fast=UpperCAmelCase			)
         self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)

     with tempfile.TemporaryDirectory() as tmp_dir:
         shutil.copy('./tests/fixtures/vocab.json'    ,  os.path.join(UpperCAmelCase    ,  'vocab.json'			)			)
         shutil.copy('./tests/fixtures/merges.txt'    ,  os.path.join(UpperCAmelCase    ,  'merges.txt'			)			)

         lowerCamelCase__					:   Optional[Any]      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  tokenizer_type='gpt2'    ,  use_fast=UpperCAmelCase			)
         self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)
 @require_tokenizers
 def   A_			(   self						:   Any			)    ->			Dict:
     with tempfile.TemporaryDirectory() as tmp_dir:
         shutil.copy('./tests/fixtures/vocab.txt'    ,  os.path.join(UpperCAmelCase    ,  'vocab.txt'			)			)

         lowerCamelCase__					:   List[Any]      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  tokenizer_type='bert'			)
         self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)

     with tempfile.TemporaryDirectory() as tmp_dir:
         shutil.copy('./tests/fixtures/vocab.json'    ,  os.path.join(UpperCAmelCase    ,  'vocab.json'			)			)
         shutil.copy('./tests/fixtures/merges.txt'    ,  os.path.join(UpperCAmelCase    ,  'merges.txt'			)			)

         lowerCamelCase__					:   Dict      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  tokenizer_type='gpt2'			)
         self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)
 def   A_			(   self						:   List[Any]			)    ->			Tuple:
     with pytest.raises(UpperCAmelCase			):
         AutoTokenizer.from_pretrained('./'    ,  tokenizer_type='xxx'			)
 @require_tokenizers
 def   A_			(   self						:   List[str]			)    ->			List[Any]:
     for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
         lowerCamelCase__					:   str      				=	tokenizer_class.from_pretrained('wietsedv/bert-base-dutch-cased'			)
         self.assertIsInstance(UpperCAmelCase    ,  (BertTokenizer, BertTokenizerFast)			)

         if isinstance(UpperCAmelCase    ,  UpperCAmelCase			):
             self.assertEqual(tokenizer.basic_tokenizer.do_lower_case    ,  UpperCAmelCase			)
         else:
             self.assertEqual(tokenizer.do_lower_case    ,  UpperCAmelCase			)

         self.assertEqual(tokenizer.model_max_length    ,  512			)
 @require_tokenizers
 def   A_			(   self						:   Tuple			)    ->			List[Any]:
     for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
         with self.assertRaisesRegex(
             UpperCAmelCase    ,  'julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'    ,  ):
             lowerCamelCase__					:   Optional[int]      				=	tokenizer_class.from_pretrained('julien-c/herlolip-not-exists'			)
 def   A_			(   self						:   List[Any]			)    ->			List[Any]:
     # tests: https://github.com/huggingface/transformers/pull/13251
     # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
     # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
     lowerCamelCase__					:   Optional[Any]      				=	TOKENIZER_MAPPING.values()
     lowerCamelCase__					:   List[Any]      				=	[]

     for slow_tok, fast_tok in tokenizers:
         if slow_tok is not None:
             tokenizer_names.append(slow_tok.__name__			)

         if fast_tok is not None:
             tokenizer_names.append(fast_tok.__name__			)

     for tokenizer_name in tokenizer_names:
         # must find the right class
         tokenizer_class_from_name(UpperCAmelCase			)
 @require_tokenizers
 def   A_			(   self						:   Tuple			)    ->			Dict:
     self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'    ,  use_fast=UpperCAmelCase			)    ,  UpperCAmelCase			)
     self.assertIsInstance(AutoTokenizer.from_pretrained('bert-base-cased'			)    ,  UpperCAmelCase			)
 @require_tokenizers
 def   A_			(   self						:   Tuple			)    ->			Dict:
     lowerCamelCase__					:   Optional[int]      				=	AutoTokenizer.from_pretrained('distilbert-base-uncased'    ,  do_lower_case=UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	'Hello, world. How are you?'
     lowerCamelCase__					:   Optional[int]      				=	tokenizer.tokenize(UpperCAmelCase			)
     self.assertEqual('[UNK]'    ,  tokens[0]			)

     lowerCamelCase__					:   List[str]      				=	AutoTokenizer.from_pretrained('microsoft/mpnet-base'    ,  do_lower_case=UpperCAmelCase			)
     lowerCamelCase__					:   List[Any]      				=	tokenizer.tokenize(UpperCAmelCase			)
     self.assertEqual('[UNK]'    ,  tokens[0]			)
 @require_tokenizers
 def   A_			(   self						:   Optional[int]			)    ->			Optional[int]:
     lowerCamelCase__					:   str      				=	AutoTokenizer.from_pretrained('robot-test/dummy-tokenizer-fast-with-model-config'			)
     self.assertEqual(type(UpperCAmelCase			)    ,  UpperCAmelCase			)
     self.assertEqual(tokenizer.model_max_length    ,  512			)
     self.assertEqual(tokenizer.vocab_size    ,  30000			)
     self.assertEqual(tokenizer.unk_token    ,  '[UNK]'			)
     self.assertEqual(tokenizer.padding_side    ,  'right'			)
     self.assertEqual(tokenizer.truncation_side    ,  'right'			)
 def   A_			(   self						:   int			)    ->			str:
     lowerCamelCase__					:   Tuple      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
     self.assertIsInstance(UpperCAmelCase    ,  (BertTokenizer, BertTokenizerFast)			)
     with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(UpperCAmelCase			)
         lowerCamelCase__					:   Any      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)

     self.assertIsInstance(UpperCAmelCase    ,  tokenizer.__class__			)
     self.assertEqual(tokenizera.vocab_size    ,  12			)
 def   A_			(   self						:   List[str]			)    ->			Optional[Any]:
     lowerCamelCase__					:   Tuple      				=	AutoTokenizer.from_pretrained('ctrl'			)
     # There is no fast CTRL so this always gives us a slow tokenizer.
     self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)
 def   A_			(   self						:   str			)    ->			Optional[Any]:
     # Check we can load the tokenizer config of an online model.
     lowerCamelCase__					:   Union[str, Any]      				=	get_tokenizer_config('bert-base-cased'			)
     lowerCamelCase__					:   List[str]      				=	config.pop('_commit_hash'    ,  UpperCAmelCase			)
     # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
     self.assertEqual(UpperCAmelCase    ,  {'do_lower_case': False}			)

     # This model does not have a tokenizer_config so we get back an empty dict.
     lowerCamelCase__					:   str      				=	get_tokenizer_config(UpperCAmelCase			)
     self.assertDictEqual(UpperCAmelCase    ,  {}			)

     # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
     lowerCamelCase__					:   Any      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
     with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(UpperCAmelCase			)
         lowerCamelCase__					:   Union[str, Any]      				=	get_tokenizer_config(UpperCAmelCase			)

     # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
     self.assertEqual(config['tokenizer_class']    ,  'BertTokenizer'			)
 def   A_			(   self						:   Dict			)    ->			Tuple:
     try:
         AutoConfig.register('custom'    ,  UpperCAmelCase			)

         AutoTokenizer.register(UpperCAmelCase    ,  slow_tokenizer_class=UpperCAmelCase			)
         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(UpperCAmelCase			):
             AutoTokenizer.register(UpperCAmelCase    ,  slow_tokenizer_class=UpperCAmelCase			)

         lowerCamelCase__					:   Optional[Any]      				=	CustomTokenizer.from_pretrained(UpperCAmelCase			)
         with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(UpperCAmelCase			)

             lowerCamelCase__					:   Optional[Any]      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
             self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]
 @require_tokenizers
 def   A_			(   self						:   Optional[Any]			)    ->			int:
     try:
         AutoConfig.register('custom'    ,  UpperCAmelCase			)

         # Can register in two steps
         AutoTokenizer.register(UpperCAmelCase    ,  slow_tokenizer_class=UpperCAmelCase			)
         self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,  (CustomTokenizer, None)			)
         AutoTokenizer.register(UpperCAmelCase    ,  fast_tokenizer_class=UpperCAmelCase			)
         self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,  (CustomTokenizer, CustomTokenizerFast)			)

         del TOKENIZER_MAPPING._extra_content[CustomConfig]
         # Can register in one step
         AutoTokenizer.register(
             UpperCAmelCase    ,  slow_tokenizer_class=UpperCAmelCase    ,  fast_tokenizer_class=UpperCAmelCase			)
         self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,  (CustomTokenizer, CustomTokenizerFast)			)

         # Trying to register something existing in the Transformers library will raise an error
         with self.assertRaises(UpperCAmelCase			):
             AutoTokenizer.register(UpperCAmelCase    ,  fast_tokenizer_class=UpperCAmelCase			)

         # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
         # and that model does not have a tokenizer.json
         with tempfile.TemporaryDirectory() as tmp_dir:
             lowerCamelCase__					:   List[Any]      				=	BertTokenizerFast.from_pretrained(UpperCAmelCase			)
             bert_tokenizer.save_pretrained(UpperCAmelCase			)
             lowerCamelCase__					:   Optional[Any]      				=	CustomTokenizerFast.from_pretrained(UpperCAmelCase			)

         with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(UpperCAmelCase			)

             lowerCamelCase__					:   Any      				=	AutoTokenizer.from_pretrained(UpperCAmelCase			)
             self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)

             lowerCamelCase__					:   Union[str, Any]      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  use_fast=UpperCAmelCase			)
             self.assertIsInstance(UpperCAmelCase    ,  UpperCAmelCase			)

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]
 def   A_			(   self						:   str			)    ->			Optional[Any]:
     # If remote code is not set, we will time out when asking whether to load the model.
     with self.assertRaises(UpperCAmelCase			):
         lowerCamelCase__					:   Optional[Any]      				=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'			)
     # If remote code is disabled, we can't load this config.
     with self.assertRaises(UpperCAmelCase			):
         lowerCamelCase__					:   Union[str, Any]      				=	AutoTokenizer.from_pretrained(
             'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=UpperCAmelCase			)

     lowerCamelCase__					:   Optional[int]      				=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=UpperCAmelCase			)
     self.assertTrue(tokenizer.special_attribute_present			)
     # Test tokenizer can be reloaded.
     with tempfile.TemporaryDirectory() as tmp_dir:
         tokenizer.save_pretrained(UpperCAmelCase			)
         lowerCamelCase__					:   List[str]      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  trust_remote_code=UpperCAmelCase			)
     self.assertTrue(reloaded_tokenizer.special_attribute_present			)

     if is_tokenizers_available():
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'			)
         self.assertEqual(reloaded_tokenizer.__class__.__name__    ,  'NewTokenizerFast'			)

         # Test we can also load the slow version
         lowerCamelCase__					:   Union[str, Any]      				=	AutoTokenizer.from_pretrained(
             'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=UpperCAmelCase    ,  use_fast=UpperCAmelCase			)
         self.assertTrue(tokenizer.special_attribute_present			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'			)
         # Test tokenizer can be reloaded.
         with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(UpperCAmelCase			)
             lowerCamelCase__					:   Any      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  trust_remote_code=UpperCAmelCase    ,  use_fast=UpperCAmelCase			)
         self.assertEqual(reloaded_tokenizer.__class__.__name__    ,  'NewTokenizer'			)
         self.assertTrue(reloaded_tokenizer.special_attribute_present			)
     else:
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'			)
         self.assertEqual(reloaded_tokenizer.__class__.__name__    ,  'NewTokenizer'			)
 @require_tokenizers
 def   A_			(   self						:   Dict			)    ->			List[Any]:

     class      lowerCAmelCase							(      __UpperCamelCase    ):
      UpperCAmelCase__									=	False


     class      lowerCAmelCase							(      __UpperCamelCase    ):
      UpperCAmelCase__									=	NewTokenizer
      UpperCAmelCase__									=	False

     try:
         AutoConfig.register('custom'    ,  UpperCAmelCase			)
         AutoTokenizer.register(UpperCAmelCase    ,  slow_tokenizer_class=UpperCAmelCase			)
         AutoTokenizer.register(UpperCAmelCase    ,  fast_tokenizer_class=UpperCAmelCase			)
         # If remote code is not set, the default is to use local
         lowerCamelCase__					:   int      				=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'			)
         self.assertFalse(tokenizer.special_attribute_present			)
         lowerCamelCase__					:   List[str]      				=	AutoTokenizer.from_pretrained('hf-internal-testing/test_dynamic_tokenizer'    ,  use_fast=UpperCAmelCase			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'			)
         self.assertFalse(tokenizer.special_attribute_present			)

         # If remote code is disabled, we load the local one.
         lowerCamelCase__					:   Union[str, Any]      				=	AutoTokenizer.from_pretrained(
             'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=UpperCAmelCase			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'			)
         self.assertFalse(tokenizer.special_attribute_present			)
         lowerCamelCase__					:   List[Any]      				=	AutoTokenizer.from_pretrained(
             'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=UpperCAmelCase    ,  use_fast=UpperCAmelCase			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'			)
         self.assertFalse(tokenizer.special_attribute_present			)

         # If remote is enabled, we load from the Hub
         lowerCamelCase__					:   Tuple      				=	AutoTokenizer.from_pretrained(
             'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=UpperCAmelCase			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'			)
         self.assertTrue(tokenizer.special_attribute_present			)
         lowerCamelCase__					:   Any      				=	AutoTokenizer.from_pretrained(
             'hf-internal-testing/test_dynamic_tokenizer'    ,  trust_remote_code=UpperCAmelCase    ,  use_fast=UpperCAmelCase			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'			)
         self.assertTrue(tokenizer.special_attribute_present			)

     finally:
         if "custom" in CONFIG_MAPPING._extra_content:
             del CONFIG_MAPPING._extra_content["custom"]
         if CustomConfig in TOKENIZER_MAPPING._extra_content:
             del TOKENIZER_MAPPING._extra_content[CustomConfig]
 def   A_			(   self						:   Any			)    ->			Dict:
     lowerCamelCase__					:   Union[str, Any]      				=	AutoTokenizer.from_pretrained(
         'hf-internal-testing/test_dynamic_tokenizer_legacy'    ,  trust_remote_code=UpperCAmelCase			)
     self.assertTrue(tokenizer.special_attribute_present			)
     if is_tokenizers_available():
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizerFast'			)

         # Test we can also load the slow version
         lowerCamelCase__					:   List[Any]      				=	AutoTokenizer.from_pretrained(
             'hf-internal-testing/test_dynamic_tokenizer_legacy'    ,  trust_remote_code=UpperCAmelCase    ,  use_fast=UpperCAmelCase			)
         self.assertTrue(tokenizer.special_attribute_present			)
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'			)
     else:
         self.assertEqual(tokenizer.__class__.__name__    ,  'NewTokenizer'			)
 def   A_			(   self						:   Optional[Any]			)    ->			int:
     with self.assertRaisesRegex(
         UpperCAmelCase    ,  'bert-base is not a local folder and is not a valid model identifier'			):
         lowerCamelCase__					:   Optional[Any]      				=	AutoTokenizer.from_pretrained('bert-base'			)
 def   A_			(   self						:   List[str]			)    ->			Dict:
     with self.assertRaisesRegex(
         UpperCAmelCase    ,  R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'			):
         lowerCamelCase__					:   Any      				=	AutoTokenizer.from_pretrained(UpperCAmelCase    ,  revision='aaaaaa'			)







 def   A_			(   self						:   str			)    ->			str:
     # Make sure we have cached the tokenizer.
     lowerCamelCase__					:   List[Any]      				=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'			)
     with RequestCounter() as counter:
         lowerCamelCase__					:   int      				=	AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'			)
         self.assertEqual(counter.get_request_count    ,  0			)
         self.assertEqual(counter.head_request_count    ,  1			)
         self.assertEqual(counter.other_request_count    ,  0			)












from typing import TYPE_CHECKING

# rely on isort to merge the imports
from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCAmelCase       : List[str]			     =       {
    """configuration_autoformer""": [
        """AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP""",
        """AutoformerConfig""",
    ],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    _UpperCAmelCase       : List[Any]			     =       [
        """AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST""",
        """AutoformerForPrediction""",
        """AutoformerModel""",
        """AutoformerPreTrainedModel""",
    ]


if TYPE_CHECKING:
    from .configuration_autoformer import (
        AUTOFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP,
        AutoformerConfig,
    )

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_autoformer import (
            AUTOFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
            AutoformerForPrediction,
            AutoformerModel,
            AutoformerPreTrainedModel,
        )

else:
    import sys

    _UpperCAmelCase       : Dict			     =       _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)





