





from transformers import BertTokenizer, EncoderDecoderModel, SeqaSeqTrainer, SeqaSeqTrainingArguments
from transformers.testing_utils import TestCasePlus, require_torch, slow
from transformers.utils import is_datasets_available


if is_datasets_available():
     import datasets



class 							_UpperCAmelCase       (       _UpperCamelCase       ):

  """simple docstring"""


  @slow
  @require_torch
  def 							lowercase						(							self					:						Tuple	)				->		Any:
     __lowerCAmelCase				      =      EncoderDecoderModel.from_encoder_decoder_pretrained('prajjwal1/bert-tiny'   ,    'prajjwal1/bert-tiny'	)
     __lowerCAmelCase				      =      BertTokenizer.from_pretrained('bert-base-uncased'	)

     __lowerCAmelCase				      =      bertabert.config.encoder.vocab_size
     __lowerCAmelCase				      =      tokenizer.sep_token_id
     __lowerCAmelCase				      =      tokenizer.cls_token_id
     __lowerCAmelCase				      =      1_2_8

     __lowerCAmelCase				      =      datasets.load_dataset('cnn_dailymail'   ,    '3.0.0'   ,    split='train[:1%]'	)
     __lowerCAmelCase				      =      datasets.load_dataset('cnn_dailymail'   ,    '3.0.0'   ,    split='validation[:1%]'	)

     __lowerCAmelCase				      =      train_dataset.select(range(3_2	)	)
     __lowerCAmelCase				      =      val_dataset.select(range(1_6	)	)

     __lowerCAmelCase				      =      4

     def _map_to_encoder_decoder_inputs(lowerCAmelCase_					:						Optional[int]	):
        # Tokenizer will automatically set [BOS] <text> [EOS]
        __lowerCAmelCase				      =      tokenizer(batch['article']   ,    padding='max_length'   ,    truncation=lowerCAmelCase_   ,    max_length=5_1_2	)
        __lowerCAmelCase				      =      tokenizer(batch['highlights']   ,    padding='max_length'   ,    truncation=lowerCAmelCase_   ,    max_length=1_2_8	)
        __lowerCAmelCase				      =      inputs.input_ids
        __lowerCAmelCase				      =      inputs.attention_mask

        __lowerCAmelCase				      =      outputs.input_ids
        __lowerCAmelCase				      =      outputs.input_ids.copy()
        __lowerCAmelCase				      =      [
            [-1_0_0 if token == tokenizer.pad_token_id else token for token in labels] for labels in batch['labels']
        ]
        __lowerCAmelCase				      =      outputs.attention_mask

        assert all(len(lowerCAmelCase_	) == 5_1_2 for x in inputs.input_ids	)
        assert all(len(lowerCAmelCase_	) == 1_2_8 for x in outputs.input_ids	)

        return batch

     def _compute_metrics(lowerCAmelCase_					:						List[str]	):
        __lowerCAmelCase				      =      pred.label_ids
        __lowerCAmelCase				      =      pred.predictions

        # all unnecessary tokens are removed
        __lowerCAmelCase				      =      tokenizer.batch_decode(lowerCAmelCase_   ,    skip_special_tokens=lowerCAmelCase_	)
        __lowerCAmelCase				      =      tokenizer.batch_decode(lowerCAmelCase_   ,    skip_special_tokens=lowerCAmelCase_	)

        __lowerCAmelCase				      =      sum([int(pred_str[i] == label_str[i]	) for i in range(len(lowerCAmelCase_	)	)]	) / len(lowerCAmelCase_	)

        return {"accuracy": accuracy}

     # map train dataset
     __lowerCAmelCase				      =      train_dataset.map(
         _map_to_encoder_decoder_inputs   ,    batched=lowerCAmelCase_   ,    batch_size=lowerCAmelCase_   ,    remove_columns=['article', 'highlights']   ,    )
     train_dataset.set_format(
         type='torch'   ,    columns=['input_ids', 'attention_mask', 'decoder_input_ids', 'decoder_attention_mask', 'labels']   ,    )

     # same for validation dataset
     __lowerCAmelCase				      =      val_dataset.map(
         _map_to_encoder_decoder_inputs   ,    batched=lowerCAmelCase_   ,    batch_size=lowerCAmelCase_   ,    remove_columns=['article', 'highlights']   ,    )
     val_dataset.set_format(
         type='torch'   ,    columns=['input_ids', 'attention_mask', 'decoder_input_ids', 'decoder_attention_mask', 'labels']   ,    )

     __lowerCAmelCase				      =      self.get_auto_remove_tmp_dir()

     __lowerCAmelCase				      =      SeqaSeqTrainingArguments(
         output_dir=lowerCAmelCase_   ,    per_device_train_batch_size=lowerCAmelCase_   ,    per_device_eval_batch_size=lowerCAmelCase_   ,    predict_with_generate=lowerCAmelCase_   ,    evaluation_strategy='steps'   ,    do_train=lowerCAmelCase_   ,    do_eval=lowerCAmelCase_   ,    warmup_steps=0   ,    eval_steps=2   ,    logging_steps=2   ,    )

     # instantiate trainer
     __lowerCAmelCase				      =      SeqaSeqTrainer(
         model=lowerCAmelCase_   ,    args=lowerCAmelCase_   ,    compute_metrics=_compute_metrics   ,    train_dataset=lowerCAmelCase_   ,    eval_dataset=lowerCAmelCase_   ,    tokenizer=lowerCAmelCase_   ,    )

     # start training
     trainer.train()








import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast





class UpperCAmelCase_				( unittest.TestCase ):


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       0


  @slow
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )

       for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(GPTaTokenizer, GPTaTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(RobertaTokenizer, RobertaTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,2_0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoConfig.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,__snake_case )
       # Check that tokenizer_type ≠ model_type
       A_          =       AutoTokenizer.from_pretrained(__snake_case ,config=__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' )
            self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with pytest.raises(__snake_case ):
            AutoTokenizer.from_pretrained('''./''' ,tokenizer_type='''xxx''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            A_          =       tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''' )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )

            if isinstance(__snake_case ,__snake_case ):
                 self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,__snake_case )
            else:
                 self.assertEqual(tokenizer.do_lower_case ,__snake_case )

            self.assertEqual(tokenizer.model_max_length ,5_1_2 )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            with self.assertRaisesRegex(
                __snake_case ,'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier''' ,):
                 A_          =       tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       TOKENIZER_MAPPING.values()
       A_          =       []

       for slow_tok, fast_tok in tokenizers:
            if slow_tok is not None:
                 tokenizer_names.append(slow_tok.__name__ )

            if fast_tok is not None:
                 tokenizer_names.append(fast_tok.__name__ )

       for tokenizer_name in tokenizer_names:
            # must find the right class
            tokenizer_class_from_name(__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ,use_fast=__snake_case ) ,__snake_case )
       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ) ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''distilbert-base-uncased''' ,do_lower_case=__snake_case )
       A_          =       '''Hello, world. How are you?'''
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )

       A_          =       AutoTokenizer.from_pretrained('''microsoft/mpnet-base''' ,do_lower_case=__snake_case )
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''' )
       self.assertEqual(type(__snake_case ) ,__snake_case )
       self.assertEqual(tokenizer.model_max_length ,5_1_2 )
       self.assertEqual(tokenizer.vocab_size ,3_0_0_0_0 )
       self.assertEqual(tokenizer.unk_token ,'''[UNK]''' )
       self.assertEqual(tokenizer.padding_side ,'''right''' )
       self.assertEqual(tokenizer.truncation_side ,'''right''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case )

       self.assertIsInstance(__snake_case ,tokenizer.__class__ )
       self.assertEqual(tokenizera.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''ctrl''' )
       # There is no fast CTRL so this always gives us a slow tokenizer.
       self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       get_tokenizer_config('''bert-base-cased''' )
       A_          =       config.pop('''_commit_hash''' ,__snake_case )
       # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
       self.assertEqual(__snake_case ,{'''do_lower_case''': False} )

       # This model does not have a tokenizer_config so we get back an empty dict.
       A_          =       get_tokenizer_config(__snake_case )
       self.assertDictEqual(__snake_case ,{} )

       # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       get_tokenizer_config(__snake_case )

       # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
       self.assertEqual(config['''tokenizer_class'''] ,'''BertTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )

            A_          =       CustomTokenizer.from_pretrained(__snake_case )
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            # Can register in two steps
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, None) )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            del TOKENIZER_MAPPING._extra_content[CustomConfig]
            # Can register in one step
            AutoTokenizer.register(
                __snake_case ,slow_tokenizer_class=__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )

            # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
            # and that model does not have a tokenizer.json
            with tempfile.TemporaryDirectory() as tmp_dir:
                 A_          =       BertTokenizerFast.from_pretrained(__snake_case )
                 bert_tokenizer.save_pretrained(__snake_case )
                 A_          =       CustomTokenizerFast.from_pretrained(__snake_case )

            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,use_fast=__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )

       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case )
       self.assertTrue(reloaded_tokenizer.special_attribute_present )

       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            # Test tokenizer can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )
                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(reloaded_tokenizer.special_attribute_present )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[int]							=   False





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[Any]							=   NewTokenizer
         __lowerCAmelCase   :						Union[str, Any]							=   False

       try:
            AutoConfig.register('''custom''' ,__snake_case )
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            # If remote code is not set, the default is to use local
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote code is disabled, we load the local one.
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote is enabled, we load from the Hub
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertTrue(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(tokenizer.special_attribute_present )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(
           '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,'''bert-base is not a local folder and is not a valid model identifier''' ):
            A_          =       AutoTokenizer.from_pretrained('''bert-base''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,revision='''aaaaaa''' )




  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
       with RequestCounter() as counter:
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
            self.assertEqual(counter.get_request_count ,0 )
            self.assertEqual(counter.head_request_count ,1 )
            self.assertEqual(counter.other_request_count ,0 )



