





import argparse
import json
from pathlib import Path

import requests
import torch
from huggingface_hub import hf_hub_download
from PIL import Image

from transformers import (
    SwiftFormerConfig,
    SwiftFormerForImageClassification,
    ViTImageProcessor,
)
from transformers.utils import logging


logging.set_verbosity_info()
a__      :     Union[str, Any]         =		logging.get_logger(__name__)

a__      :     List[str]         =		torch.device('cpu')

def      UpperCAmelCase_    (    )	->				str:
     '''simple docstring'''


     A_          =       '''http://images.cocodataset.org/val2017/000000039769.jpg'''
     A_          =       Image.open(requests.get(_UpperCAmelCase  ,						stream=_UpperCAmelCase				).raw				)
     return im

def      UpperCAmelCase_    (    _UpperCAmelCase							:Optional[int]				)	->				Any:
     '''simple docstring'''


     if swiftformer_name == "swiftformer_xs":
          return torch.tensor([-2.1703e00, 2.1107e00, -2.0811e00, 8.8685e-01, 2.4360e-01]				)

     elif swiftformer_name == "swiftformer_s":
          return torch.tensor([3.9636e-01, 2.3478e-01, -1.6963e00, -1.7381e00, -8.6337e-01]				)

     elif swiftformer_name == "swiftformer_l1":
          return torch.tensor([-4.2768e-01, -4.7429e-01, -1.0897e00, -1.0248e00, 3.5523e-02]				)

     elif swiftformer_name == "swiftformer_l3":
          return torch.tensor([-2.5330e-01, 2.4211e-01, -6.0185e-01, -8.2789e-01, -6.0446e-02]				)

def      UpperCAmelCase_    (    _UpperCAmelCase							:Optional[Any]  ,						_UpperCAmelCase							:List[Any]  ,						_UpperCAmelCase							:Optional[Any]				)	->				Optional[Any]:
     '''simple docstring'''


     A_          =       dct.pop(_UpperCAmelCase				)
     A_          =       val

def      UpperCAmelCase_    (    _UpperCAmelCase							:Union[str, Any]				)	->				Union[str, Any]:
     '''simple docstring'''


     A_          =       []
     for k in state_dict.keys():
          A_          =       k
          if ".pwconv" in k:
               A_          =       k_new.replace('''.pwconv'''  ,						'''.point_wise_conv'''				)
          if ".dwconv" in k:
               A_          =       k_new.replace('''.dwconv'''  ,						'''.depth_wise_conv'''				)
          if ".Proj." in k:
               A_          =       k_new.replace('''.Proj.'''  ,						'''.proj.'''				)
          if "patch_embed" in k_new:
               A_          =       k_new.replace('''patch_embed'''  ,						'''swiftformer.patch_embed.patch_embedding'''				)
          if "network" in k_new:
               A_          =       k_new.split('''.'''				)
               if ls[2].isdigit():
                    A_          =       '''swiftformer.encoder.network.''' + ls[1] + '''.blocks.''' + ls[2] + '''.''' + '''.'''.join(ls[3:]				)
               else:
                    A_          =       k_new.replace('''network'''  ,						'''swiftformer.encoder.network'''				)
          rename_keys.append((k, k_new)				)
     return rename_keys

@torch.no_grad()
def      UpperCAmelCase_    (    _UpperCAmelCase							:Tuple  ,						_UpperCAmelCase							:Union[str, Any]  ,						_UpperCAmelCase							:int				)	->				Union[str, Any]:
     '''simple docstring'''


     A_          =       SwiftFormerConfig()

     # dataset (ImageNet-21k only or also fine-tuned on ImageNet 2012), patch_size and image_size
     A_          =       1000
     A_          =       '''huggingface/label-files'''
     A_          =       '''imagenet-1k-id2label.json'''
     A_          =       json.load(open(hf_hub_download(_UpperCAmelCase  ,						_UpperCAmelCase  ,						repo_type='''dataset'''				)  ,						'''r'''				)				)
     A_          =       {int(_UpperCAmelCase				): v for k, v in idalabel.items()}
     A_          =       idalabel
     A_          =       {v: k for k, v in idalabel.items()}

     # size of the architecture
     if swiftformer_name == "swiftformer_xs":
          A_          =       [3, 3, 6, 4]
          A_          =       [48, 56, 112, 220]

     elif swiftformer_name == "swiftformer_s":
          A_          =       [3, 3, 9, 6]
          A_          =       [48, 64, 168, 224]

     elif swiftformer_name == "swiftformer_l1":
          A_          =       [4, 3, 10, 5]
          A_          =       [48, 96, 192, 384]

     elif swiftformer_name == "swiftformer_l3":
          A_          =       [4, 4, 12, 6]
          A_          =       [64, 128, 320, 512]

     # load state_dict of original model, remove and rename some keys
     if original_ckpt:
          if original_ckpt.startswith('''https'''				):
               A_          =       torch.hub.load_state_dict_from_url(_UpperCAmelCase  ,						map_location='''cpu'''  ,						check_hash=_UpperCAmelCase				)
          else:
               A_          =       torch.load(_UpperCAmelCase  ,						map_location='''cpu'''				)
     A_          =       checkpoint

     A_          =       create_rename_keys(_UpperCAmelCase				)
     for rename_key_src, rename_key_dest in rename_keys:
          rename_key(_UpperCAmelCase  ,						_UpperCAmelCase  ,						_UpperCAmelCase				)

     # load HuggingFace model
     A_          =       SwiftFormerForImageClassification(_UpperCAmelCase				).eval()
     hf_model.load_state_dict(_UpperCAmelCase				)

     # prepare test inputs
     A_          =       prepare_img()
     A_          =       ViTImageProcessor.from_pretrained('''preprocessor_config'''				)
     A_          =       processor(images=_UpperCAmelCase  ,						return_tensors='''pt'''				)

     # compare outputs from both models
     A_          =       get_expected_output(_UpperCAmelCase				)
     A_          =       hf_model(inputs['''pixel_values''']				).logits

     assert hf_logits.shape == torch.Size([1, 1000]				)
     assert torch.allclose(hf_logits[0, 0:5]  ,						_UpperCAmelCase  ,						atol=1e-3				)

     Path(_UpperCAmelCase				).mkdir(exist_ok=_UpperCAmelCase				)
     print(f'Saving model {swiftformer_name} to {pytorch_dump_folder_path}'				)
     hf_model.save_pretrained(_UpperCAmelCase				)


if __name__ == "__main__":
      a__      :     List[Any]         =		argparse.ArgumentParser()
      # Required parameters
      parser.add_argument(
          '--swiftformer_name',
          default='swiftformer_xs',
          choices=['swiftformer_xs', 'swiftformer_s', 'swiftformer_l1', 'swiftformer_l3'],
          type=str,
          help='Name of the SwiftFormer model you\'d like to convert.',
      )
      parser.add_argument(
          '--pytorch_dump_folder_path',
          default='./converted_outputs/',
          type=str,
          help='Path to the output PyTorch model directory.',
      )
      parser.add_argument('--original_ckpt', default=None, type=str, help='Path to the original model checkpoint.')

      a__      :     List[str]         =		parser.parse_args()
      convert_swiftformer_checkpoint(args.swiftformer_name, args.pytorch_dump_folder_path, args.original_ckpt)










import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast





class UpperCAmelCase_				( unittest.TestCase ):


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       0


  @slow
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )

       for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(GPTaTokenizer, GPTaTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(RobertaTokenizer, RobertaTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,2_0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoConfig.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,__snake_case )
       # Check that tokenizer_type ≠ model_type
       A_          =       AutoTokenizer.from_pretrained(__snake_case ,config=__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' )
            self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with pytest.raises(__snake_case ):
            AutoTokenizer.from_pretrained('''./''' ,tokenizer_type='''xxx''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            A_          =       tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''' )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )

            if isinstance(__snake_case ,__snake_case ):
                 self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,__snake_case )
            else:
                 self.assertEqual(tokenizer.do_lower_case ,__snake_case )

            self.assertEqual(tokenizer.model_max_length ,5_1_2 )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            with self.assertRaisesRegex(
                __snake_case ,'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier''' ,):
                 A_          =       tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       TOKENIZER_MAPPING.values()
       A_          =       []

       for slow_tok, fast_tok in tokenizers:
            if slow_tok is not None:
                 tokenizer_names.append(slow_tok.__name__ )

            if fast_tok is not None:
                 tokenizer_names.append(fast_tok.__name__ )

       for tokenizer_name in tokenizer_names:
            # must find the right class
            tokenizer_class_from_name(__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ,use_fast=__snake_case ) ,__snake_case )
       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ) ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''distilbert-base-uncased''' ,do_lower_case=__snake_case )
       A_          =       '''Hello, world. How are you?'''
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )

       A_          =       AutoTokenizer.from_pretrained('''microsoft/mpnet-base''' ,do_lower_case=__snake_case )
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''' )
       self.assertEqual(type(__snake_case ) ,__snake_case )
       self.assertEqual(tokenizer.model_max_length ,5_1_2 )
       self.assertEqual(tokenizer.vocab_size ,3_0_0_0_0 )
       self.assertEqual(tokenizer.unk_token ,'''[UNK]''' )
       self.assertEqual(tokenizer.padding_side ,'''right''' )
       self.assertEqual(tokenizer.truncation_side ,'''right''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case )

       self.assertIsInstance(__snake_case ,tokenizer.__class__ )
       self.assertEqual(tokenizera.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''ctrl''' )
       # There is no fast CTRL so this always gives us a slow tokenizer.
       self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       get_tokenizer_config('''bert-base-cased''' )
       A_          =       config.pop('''_commit_hash''' ,__snake_case )
       # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
       self.assertEqual(__snake_case ,{'''do_lower_case''': False} )

       # This model does not have a tokenizer_config so we get back an empty dict.
       A_          =       get_tokenizer_config(__snake_case )
       self.assertDictEqual(__snake_case ,{} )

       # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       get_tokenizer_config(__snake_case )

       # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
       self.assertEqual(config['''tokenizer_class'''] ,'''BertTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )

            A_          =       CustomTokenizer.from_pretrained(__snake_case )
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            # Can register in two steps
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, None) )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            del TOKENIZER_MAPPING._extra_content[CustomConfig]
            # Can register in one step
            AutoTokenizer.register(
                __snake_case ,slow_tokenizer_class=__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )

            # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
            # and that model does not have a tokenizer.json
            with tempfile.TemporaryDirectory() as tmp_dir:
                 A_          =       BertTokenizerFast.from_pretrained(__snake_case )
                 bert_tokenizer.save_pretrained(__snake_case )
                 A_          =       CustomTokenizerFast.from_pretrained(__snake_case )

            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,use_fast=__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )

       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case )
       self.assertTrue(reloaded_tokenizer.special_attribute_present )

       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            # Test tokenizer can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )
                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(reloaded_tokenizer.special_attribute_present )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[int]							=   False





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[Any]							=   NewTokenizer
         __lowerCAmelCase   :						Union[str, Any]							=   False

       try:
            AutoConfig.register('''custom''' ,__snake_case )
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            # If remote code is not set, the default is to use local
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote code is disabled, we load the local one.
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote is enabled, we load from the Hub
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertTrue(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(tokenizer.special_attribute_present )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(
           '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,'''bert-base is not a local folder and is not a valid model identifier''' ):
            A_          =       AutoTokenizer.from_pretrained('''bert-base''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,revision='''aaaaaa''' )




  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
       with RequestCounter() as counter:
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
            self.assertEqual(counter.get_request_count ,0 )
            self.assertEqual(counter.head_request_count ,1 )
            self.assertEqual(counter.other_request_count ,0 )



