





import os
from dataclasses import dataclass, field
from io import BytesIO
from typing import TYPE_CHECKING, Any, ClassVar, Dict, Optional, Union

import numpy as np
import pyarrow as pa

from .. import config
from ..download.streaming_download_manager import xopen, xsplitext
from ..table import array_cast
from ..utils.py_utils import no_op_if_value_is_null, string_to_dict


if TYPE_CHECKING:
      from .features import FeatureType


a__   ,				a__   ,				a__      :     List[str]         =		False, False, False





@dataclass
class UpperCAmelCase_				:
  __lowerCAmelCase   :						Optional[int]							=   None
  __lowerCAmelCase   :						bool							=   True
  __lowerCAmelCase   :						bool							=   True
  __lowerCAmelCase   :						Optional[str]							=   None
  # Automatically constructed
  __lowerCAmelCase   :						ClassVar[str]							=   "dict"
  __lowerCAmelCase   :						ClassVar[Any]							=   pa.struct({"""bytes""": pa.binary(), """path""": pa.string()} )
  __lowerCAmelCase   :						str							=   field(default="""Audio"""     ,		init=_SCREAMING_SNAKE_CASE     ,		repr=_SCREAMING_SNAKE_CASE )


  def __call__(							self ):



       """simple docstring"""





       return self.pa_type


  def 						__UpperCAmelCase     (							self ,__snake_case ):



       """simple docstring"""





       try:
            import soundfile as sf  # soundfile is a dependency of librosa, needed to decode audio files.
       except ImportError as err:
            raise ImportError('''To support encoding audio data, please install \'soundfile\'.''' ) from err
       if isinstance(__snake_case ,__snake_case ):
            return {"bytes": None, "path": value}
       elif isinstance(__snake_case ,__snake_case ):
            return {"bytes": value, "path": None}
       elif "array" in value:
            # convert the audio array to wav bytes
            A_          =       BytesIO()
            sf.write(__snake_case ,value['''array'''] ,value['''sampling_rate'''] ,format='''wav''' )
            return {"bytes": buffer.getvalue(), "path": None}
       elif value.get('''path''' ) is not None and os.path.isfile(value['''path'''] ):
            # we set "bytes": None to not duplicate the data if they're already available locally
            if value["path"].endswith('''pcm''' ):
                 # "PCM" only has raw audio bytes
                 if value.get('''sampling_rate''' ) is None:
                      # At least, If you want to convert "PCM-byte" to "WAV-byte", you have to know sampling rate
                      raise KeyError('''To use PCM files, please specify a \'sampling_rate\' in Audio object''' )
                 if value.get('''bytes''' ):
                      # If we already had PCM-byte, we don`t have to make "read file, make bytes" (just use it!)
                      A_          =       np.frombuffer(value['''bytes'''] ,dtype=np.intaa ).astype(np.floataa ) / 3_2_7_6_7
                 else:
                      A_          =       np.memmap(value['''path'''] ,dtype='''h''' ,mode='''r''' ).astype(np.floataa ) / 3_2_7_6_7

                 A_          =       BytesIO(bytes() )
                 sf.write(__snake_case ,__snake_case ,value['''sampling_rate'''] ,format='''wav''' )
                 return {"bytes": buffer.getvalue(), "path": None}
            else:
                 return {"bytes": None, "path": value.get('''path''' )}
       elif value.get('''bytes''' ) is not None or value.get('''path''' ) is not None:
            # store the audio bytes, and path is used to infer the audio format using the file extension
            return {"bytes": value.get('''bytes''' ), "path": value.get('''path''' )}
       else:
            raise ValueError(
                f'An audio sample should have one of \'path\' or \'bytes\' but they are missing or None in {value}.' )


  def 						__UpperCAmelCase     (							self ,__snake_case ,__snake_case = None ):



       """simple docstring"""





       if not self.decode:
            raise RuntimeError('''Decoding is disabled for this feature. Please use Audio(decode=True) instead.''' )

       A_   ,    A_          =       (value['''path'''], BytesIO(value['''bytes'''] )) if value['''bytes'''] is not None else (value['''path'''], None)
       if path is None and file is None:
            raise ValueError(f'An audio sample should have one of \'path\' or \'bytes\' but both are None in {value}.' )

       try:
            import librosa
            import soundfile as sf
       except ImportError as err:
            raise ImportError('''To support decoding audio files, please install \'librosa\' and \'soundfile\'.''' ) from err

       A_          =       xsplitext(__snake_case )[1][1:].lower() if path is not None else None
       if not config.IS_OPUS_SUPPORTED and audio_format == "opus":
            raise RuntimeError(
                '''Decoding \'opus\' files requires system library \'libsndfile\'>=1.0.31, '''
                '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ''' )
       elif not config.IS_MP3_SUPPORTED and audio_format == "mp3":
            raise RuntimeError(
                '''Decoding \'mp3\' files requires system library \'libsndfile\'>=1.1.0, '''
                '''You can try to update `soundfile` python library: `pip install "soundfile>=0.12.1"`. ''' )

       if file is None:
            A_          =       token_per_repo_id or {}
            A_          =       path.split('''::''' )[-1]
            try:
                 A_          =       string_to_dict(__snake_case ,config.HUB_DATASETS_URL )['''repo_id''']
                 A_          =       token_per_repo_id[repo_id]
            except (ValueError, KeyError):
                 A_          =       None

            with xopen(__snake_case ,'''rb''' ,use_auth_token=__snake_case ) as f:
                 A_   ,    A_          =       sf.read(__snake_case )

       else:
            A_   ,    A_          =       sf.read(__snake_case )

       A_          =       array.T
       if self.mono:
            A_          =       librosa.to_mono(__snake_case )
       if self.sampling_rate and self.sampling_rate != sampling_rate:
            A_          =       librosa.resample(__snake_case ,orig_sr=__snake_case ,target_sr=self.sampling_rate )
            A_          =       self.sampling_rate

       return {"path": path, "array": array, "sampling_rate": sampling_rate}


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       from .features import Value

       if self.decode:
            raise ValueError('''Cannot flatten a decoded Audio feature.''' )
       return {
           "bytes": Value('''binary''' ),
           "path": Value('''string''' ),
       }


  def 						__UpperCAmelCase     (							self ,__snake_case ):



       """simple docstring"""





       if pa.types.is_string(storage.type ):
            A_          =       pa.array([None] * len(__snake_case ) ,type=pa.binary() )
            A_          =       pa.StructArray.from_arrays([bytes_array, storage] ,['''bytes''', '''path'''] ,mask=storage.is_null() )
       elif pa.types.is_binary(storage.type ):
            A_          =       pa.array([None] * len(__snake_case ) ,type=pa.string() )
            A_          =       pa.StructArray.from_arrays([storage, path_array] ,['''bytes''', '''path'''] ,mask=storage.is_null() )
       elif pa.types.is_struct(storage.type ) and storage.type.get_all_field_indices('''array''' ):
            A_          =       pa.array([Audio().encode_example(__snake_case ) if x is not None else None for x in storage.to_pylist()] )
       elif pa.types.is_struct(storage.type ):
            if storage.type.get_field_index('''bytes''' ) >= 0:
                 A_          =       storage.field('''bytes''' )
            else:
                 A_          =       pa.array([None] * len(__snake_case ) ,type=pa.binary() )
            if storage.type.get_field_index('''path''' ) >= 0:
                 A_          =       storage.field('''path''' )
            else:
                 A_          =       pa.array([None] * len(__snake_case ) ,type=pa.string() )
            A_          =       pa.StructArray.from_arrays([bytes_array, path_array] ,['''bytes''', '''path'''] ,mask=storage.is_null() )
       return array_cast(__snake_case ,self.pa_type )




  def 						__UpperCAmelCase     (							self ,__snake_case ):



       """simple docstring"""





       @no_op_if_value_is_null
       def path_to_bytes(__snake_case ):
            with xopen(__snake_case ,'''rb''' ) as f:
                 A_          =       f.read()
            return bytes_

       A_          =       pa.array(
           [
               (path_to_bytes(x['''path'''] ) if x['''bytes'''] is None else x['''bytes''']) if x is not None else None
               for x in storage.to_pylist()
           ] ,type=pa.binary() ,)
       A_          =       pa.array(
           [os.path.basename(__snake_case ) if path is not None else None for path in storage.field('''path''' ).to_pylist()] ,type=pa.string() ,)
       A_          =       pa.StructArray.from_arrays([bytes_array, path_array] ,['''bytes''', '''path'''] ,mask=bytes_array.is_null() )
       return array_cast(__snake_case ,self.pa_type )










import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast





class UpperCAmelCase_				( unittest.TestCase ):


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       0


  @slow
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )

       for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
            A_          =       AutoTokenizer.from_pretrained(__snake_case )
            self.assertIsNotNone(__snake_case )
            self.assertIsInstance(__snake_case ,(GPTaTokenizer, GPTaTokenizerFast) )
            self.assertGreater(len(__snake_case ) ,0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(RobertaTokenizer, RobertaTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,2_0 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoConfig.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,__snake_case )
       # Check that tokenizer_type ≠ model_type
       A_          =       AutoTokenizer.from_pretrained(__snake_case ,config=__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       self.assertEqual(tokenizer.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' ,use_fast=__snake_case )
            self.assertIsInstance(__snake_case ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.txt''' ,os.path.join(__snake_case ,'''vocab.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''bert''' )
            self.assertIsInstance(__snake_case ,__snake_case )

       with tempfile.TemporaryDirectory() as tmp_dir:
            shutil.copy('''./tests/fixtures/vocab.json''' ,os.path.join(__snake_case ,'''vocab.json''' ) )
            shutil.copy('''./tests/fixtures/merges.txt''' ,os.path.join(__snake_case ,'''merges.txt''' ) )

            A_          =       AutoTokenizer.from_pretrained(__snake_case ,tokenizer_type='''gpt2''' )
            self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with pytest.raises(__snake_case ):
            AutoTokenizer.from_pretrained('''./''' ,tokenizer_type='''xxx''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            A_          =       tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased''' )
            self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )

            if isinstance(__snake_case ,__snake_case ):
                 self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,__snake_case )
            else:
                 self.assertEqual(tokenizer.do_lower_case ,__snake_case )

            self.assertEqual(tokenizer.model_max_length ,5_1_2 )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
            with self.assertRaisesRegex(
                __snake_case ,'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier''' ,):
                 A_          =       tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       TOKENIZER_MAPPING.values()
       A_          =       []

       for slow_tok, fast_tok in tokenizers:
            if slow_tok is not None:
                 tokenizer_names.append(slow_tok.__name__ )

            if fast_tok is not None:
                 tokenizer_names.append(fast_tok.__name__ )

       for tokenizer_name in tokenizer_names:
            # must find the right class
            tokenizer_class_from_name(__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ,use_fast=__snake_case ) ,__snake_case )
       self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased''' ) ,__snake_case )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''distilbert-base-uncased''' ,do_lower_case=__snake_case )
       A_          =       '''Hello, world. How are you?'''
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )

       A_          =       AutoTokenizer.from_pretrained('''microsoft/mpnet-base''' ,do_lower_case=__snake_case )
       A_          =       tokenizer.tokenize(__snake_case )
       self.assertEqual('''[UNK]''' ,tokens[0] )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config''' )
       self.assertEqual(type(__snake_case ) ,__snake_case )
       self.assertEqual(tokenizer.model_max_length ,5_1_2 )
       self.assertEqual(tokenizer.vocab_size ,3_0_0_0_0 )
       self.assertEqual(tokenizer.unk_token ,'''[UNK]''' )
       self.assertEqual(tokenizer.padding_side ,'''right''' )
       self.assertEqual(tokenizer.truncation_side ,'''right''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       self.assertIsInstance(__snake_case ,(BertTokenizer, BertTokenizerFast) )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case )

       self.assertIsInstance(__snake_case ,tokenizer.__class__ )
       self.assertEqual(tokenizera.vocab_size ,1_2 )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''ctrl''' )
       # There is no fast CTRL so this always gives us a slow tokenizer.
       self.assertIsInstance(__snake_case ,__snake_case )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       get_tokenizer_config('''bert-base-cased''' )
       A_          =       config.pop('''_commit_hash''' ,__snake_case )
       # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
       self.assertEqual(__snake_case ,{'''do_lower_case''': False} )

       # This model does not have a tokenizer_config so we get back an empty dict.
       A_          =       get_tokenizer_config(__snake_case )
       self.assertDictEqual(__snake_case ,{} )

       # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
       A_          =       AutoTokenizer.from_pretrained(__snake_case )
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       get_tokenizer_config(__snake_case )

       # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
       self.assertEqual(config['''tokenizer_class'''] ,'''BertTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )

            A_          =       CustomTokenizer.from_pretrained(__snake_case )
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       try:
            AutoConfig.register('''custom''' ,__snake_case )

            # Can register in two steps
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, None) )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            del TOKENIZER_MAPPING._extra_content[CustomConfig]
            # Can register in one step
            AutoTokenizer.register(
                __snake_case ,slow_tokenizer_class=__snake_case ,fast_tokenizer_class=__snake_case )
            self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,(CustomTokenizer, CustomTokenizerFast) )

            # Trying to register something existing in the Transformers library will raise an error
            with self.assertRaises(__snake_case ):
                 AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )

            # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
            # and that model does not have a tokenizer.json
            with tempfile.TemporaryDirectory() as tmp_dir:
                 A_          =       BertTokenizerFast.from_pretrained(__snake_case )
                 bert_tokenizer.save_pretrained(__snake_case )
                 A_          =       CustomTokenizerFast.from_pretrained(__snake_case )

            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,use_fast=__snake_case )
                 self.assertIsInstance(__snake_case ,__snake_case )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
       # If remote code is disabled, we can't load this config.
       with self.assertRaises(__snake_case ):
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )

       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       # Test tokenizer can be reloaded.
       with tempfile.TemporaryDirectory() as tmp_dir:
            tokenizer.save_pretrained(__snake_case )
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case )
       self.assertTrue(reloaded_tokenizer.special_attribute_present )

       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            # Test tokenizer can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(__snake_case )
                 A_          =       AutoTokenizer.from_pretrained(__snake_case ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(reloaded_tokenizer.special_attribute_present )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertEqual(reloaded_tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  @require_tokenizers
  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[int]							=   False





       class UpperCAmelCase_				( _SCREAMING_SNAKE_CASE ):
         __lowerCAmelCase   :						Optional[Any]							=   NewTokenizer
         __lowerCAmelCase   :						Union[str, Any]							=   False

       try:
            AutoConfig.register('''custom''' ,__snake_case )
            AutoTokenizer.register(__snake_case ,slow_tokenizer_class=__snake_case )
            AutoTokenizer.register(__snake_case ,fast_tokenizer_class=__snake_case )
            # If remote code is not set, the default is to use local
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer''' ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote code is disabled, we load the local one.
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertFalse(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertFalse(tokenizer.special_attribute_present )

            # If remote is enabled, we load from the Hub
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )
            self.assertTrue(tokenizer.special_attribute_present )
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
            self.assertTrue(tokenizer.special_attribute_present )

       finally:
            if "custom" in CONFIG_MAPPING._extra_content:
                 del CONFIG_MAPPING._extra_content["custom"]
            if CustomConfig in TOKENIZER_MAPPING._extra_content:
                 del TOKENIZER_MAPPING._extra_content[CustomConfig]


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained(
           '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case )
       self.assertTrue(tokenizer.special_attribute_present )
       if is_tokenizers_available():
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizerFast''' )

            # Test we can also load the slow version
            A_          =       AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer_legacy''' ,trust_remote_code=__snake_case ,use_fast=__snake_case )
            self.assertTrue(tokenizer.special_attribute_present )
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )
       else:
            self.assertEqual(tokenizer.__class__.__name__ ,'''NewTokenizer''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,'''bert-base is not a local folder and is not a valid model identifier''' ):
            A_          =       AutoTokenizer.from_pretrained('''bert-base''' )


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       with self.assertRaisesRegex(
           __snake_case ,R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)''' ):
            A_          =       AutoTokenizer.from_pretrained(__snake_case ,revision='''aaaaaa''' )




  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
       with RequestCounter() as counter:
            A_          =       AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert''' )
            self.assertEqual(counter.get_request_count ,0 )
            self.assertEqual(counter.head_request_count ,1 )
            self.assertEqual(counter.other_request_count ,0 )



