




'''simple docstring'''


import copy
import re

class      A       :
							lowercase_							     =							'hp'
							lowercase_							     =							{}
							lowercase_							     =							None







							@classmethod
							def   __lowerCAmelCase     (    cls			:    List[Any]				,     lowerCAmelCase_			:    Optional[int]				,     lowerCAmelCase_			:    List[Any]   )     ->				Any:


														"""simple docstring"""


														_a      	=							prefix
														_a      	=							defaults
														cls.build_naming_info()







							@staticmethod
							def   __lowerCAmelCase     (    lowerCAmelCase_			:    Any				,     lowerCAmelCase_			:    List[str]   )     ->				Union[str, Any]:


														"""simple docstring"""


														if len(lowerCAmelCase_   ) == 0:
																					return ""
														_a      	=							None
														if any(char.isdigit() for char in word   ):
																					raise Exception(F'Parameters should not contain numbers: \'{word}\' contains a number'   )
														if word in info["short_word"]:
																					return info["short_word"][word]
														for prefix_len in range(1				,     len(lowerCAmelCase_   ) + 1   ):
																					_a      	=							word[:prefix_len]
																					if prefix in info["reverse_short_word"]:
																												continue
																					else:
																												_a      	=							prefix
																												break

														if short_word is None:
																					# Paranoid fallback
																					def int_to_alphabetic(lowerCAmelCase_			:    Optional[int]   ):
																												_a      	=							''''''
																												while integer != 0:
																																			_a      	=							chr(ord('''A'''   ) + integer % 10   ) + s
																																			integer //= 10
																												return s

																					_a      	=							0
																					while True:
																												_a      	=							word + '''#''' + int_to_alphabetic(lowerCAmelCase_   )
																												if sword in info["reverse_short_word"]:
																																			continue
																												else:
																																			_a      	=							sword
																																			break

														_a      	=							short_word
														_a      	=							word
														return short_word







							@staticmethod
							def   __lowerCAmelCase     (    lowerCAmelCase_			:    List[str]				,     lowerCAmelCase_			:    Optional[int]   )     ->				int:


														"""simple docstring"""


														_a      	=							param_name.split('''_'''   )

														_a      	=							[TrialShortNamer.shortname_for_word(lowerCAmelCase_				,     lowerCAmelCase_   ) for word in words]

														# We try to create a separatorless short name, but if there is a collision we have to fallback
														# to a separated short name
														_a      	=							['''''', '''_''']

														for separator in separators:
																					_a      	=							separator.join(lowerCAmelCase_   )
																					if shortname not in info["reverse_short_param"]:
																												_a      	=							shortname
																												_a      	=							param_name
																												return shortname

														return param_name







							@staticmethod
							def   __lowerCAmelCase     (    lowerCAmelCase_			:    int				,     lowerCAmelCase_			:    Any   )     ->				Any:


														"""simple docstring"""


														_a      	=							TrialShortNamer.shortname_for_key(lowerCAmelCase_				,     lowerCAmelCase_   )
														_a      	=							short_name
														_a      	=							param_name







							@classmethod
							def   __lowerCAmelCase     (    cls			:    Dict   )     ->				Optional[int]:


														"""simple docstring"""


														if cls.NAMING_INFO is not None:
																					return

														_a      	=							{
														    '''short_word''': {},
														    '''reverse_short_word''': {},
														    '''short_param''': {},
														    '''reverse_short_param''': {},
														}

														_a      	=							list(cls.DEFAULTS.keys()   )

														for k in field_keys:
																					cls.add_new_param_name(lowerCAmelCase_				,     lowerCAmelCase_   )

														_a      	=							info







							@classmethod
							def   __lowerCAmelCase     (    cls			:    int				,     lowerCAmelCase_			:    Dict   )     ->				Optional[Any]:


														"""simple docstring"""


														cls.build_naming_info()
														assert cls.PREFIX is not None
														_a      	=							[copy.copy(cls.PREFIX   )]

														for k, v in params.items():
																					if k not in cls.DEFAULTS:
																												raise Exception(F'You should provide a default value for the param name {k} with value {v}'   )
																					if v == cls.DEFAULTS[k]:
																												# The default value is not added to the name
																												continue

																					_a      	=							cls.NAMING_INFO['''short_param'''][k]

																					if isinstance(lowerCAmelCase_				,     lowerCAmelCase_   ):
																												_a      	=							1 if v else 0

																					_a      	=							'''''' if isinstance(lowerCAmelCase_				,     (int, float)   ) else '''-'''
																					_a      	=							F'{key}{sep}{v}'
																					name.append(lowerCAmelCase_   )

														return "_".join(lowerCAmelCase_   )







							@classmethod
							def   __lowerCAmelCase     (    cls			:    List[Any]				,     lowerCAmelCase_			:    Any   )     ->				Optional[int]:


														"""simple docstring"""


														_a      	=							repr[len(cls.PREFIX   ) + 1 :]
														if repr == "":
																					_a      	=							[]
														else:
																					_a      	=							repr.split('''_'''   )

														_a      	=							{}

														for value in values:
																					if "-" in value:
																												_a	,				_a      	=							value.split('''-'''   )
																					else:
																												_a      	=							re.sub('''[0-9.]'''				,     ''''''				,     lowerCAmelCase_   )
																												_a      	=							float(re.sub('''[^0-9.]'''				,     ''''''				,     lowerCAmelCase_   )   )

																					_a      	=							cls.NAMING_INFO['''reverse_short_param'''][p_k]

																					_a      	=							p_v

														for k in cls.DEFAULTS:
																					if k not in parameters:
																												_a      	=							cls.DEFAULTS[k]

														return parameters







import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / "utils"))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
		from test_module.custom_tokenization_fast import CustomTokenizerFast

class 						__lowerCAmelCase      (      unittest.TestCase					):
							def  lowerCamelCase__				(		self		:Optional[Any]							):







								'''simple docstring'''







								a									=				0
							@slow
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)

								for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)
									self.assertIsNotNone(__magic_name__							)
									self.assertIsInstance(__magic_name__						, (GPTaTokenizer, GPTaTokenizerFast)							)
									self.assertGreater(len(__magic_name__							)						, 0							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (RobertaTokenizer, RobertaTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 20							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoConfig.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, __magic_name__							)
								# Check that tokenizer_type ≠ model_type
								a									=				AutoTokenizer.from_pretrained(__magic_name__						, config=__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								self.assertEqual(tokenizer.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""						, use_fast=__magic_name__							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.txt"""						, os.path.join(__magic_name__						, """vocab.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""bert"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)

								with tempfile.TemporaryDirectory() as tmp_dir:
									shutil.copy("""./tests/fixtures/vocab.json"""						, os.path.join(__magic_name__						, """vocab.json"""							)							)
									shutil.copy("""./tests/fixtures/merges.txt"""						, os.path.join(__magic_name__						, """merges.txt"""							)							)

									a									=				AutoTokenizer.from_pretrained(__magic_name__						, tokenizer_type="""gpt2"""							)
									self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:Dict							):







								'''simple docstring'''







								with pytest.raises(__magic_name__							):
									AutoTokenizer.from_pretrained("""./"""						, tokenizer_type="""xxx"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									a									=				tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
									self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)

									if isinstance(__magic_name__						, __magic_name__							):
										self.assertEqual(tokenizer.basic_tokenizer.do_lower_case						, __magic_name__							)
									else:
										self.assertEqual(tokenizer.do_lower_case						, __magic_name__							)

									self.assertEqual(tokenizer.model_max_length						, 512							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
									with self.assertRaisesRegex(
									    __magic_name__						, """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""						, ):
										a									=				tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				TOKENIZER_MAPPING.values()
								a									=				[]

								for slow_tok, fast_tok in tokenizers:
									if slow_tok is not None:
										tokenizer_names.append(slow_tok.__name__							)

									if fast_tok is not None:
										tokenizer_names.append(fast_tok.__name__							)

								for tokenizer_name in tokenizer_names:
									# must find the right class
									tokenizer_class_from_name(__magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""						, use_fast=__magic_name__							)						, __magic_name__							)
								self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)						, __magic_name__							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""distilbert-base-uncased"""						, do_lower_case=__magic_name__							)
								a									=				"""Hello, world. How are you?"""
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)

								a									=				AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""						, do_lower_case=__magic_name__							)
								a									=				tokenizer.tokenize(__magic_name__							)
								self.assertEqual("""[UNK]"""						, tokens[0]							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
								self.assertEqual(type(__magic_name__							)						, __magic_name__							)
								self.assertEqual(tokenizer.model_max_length						, 512							)
								self.assertEqual(tokenizer.vocab_size						, 3_0000							)
								self.assertEqual(tokenizer.unk_token						, """[UNK]"""							)
								self.assertEqual(tokenizer.padding_side						, """right"""							)
								self.assertEqual(tokenizer.truncation_side						, """right"""							)
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								self.assertIsInstance(__magic_name__						, (BertTokenizer, BertTokenizerFast)							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__							)

								self.assertIsInstance(__magic_name__						, tokenizer.__class__							)
								self.assertEqual(tokenizera.vocab_size						, 12							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""ctrl"""							)
								# There is no fast CTRL so this always gives us a slow tokenizer.
								self.assertIsInstance(__magic_name__						, __magic_name__							)
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				get_tokenizer_config("""bert-base-cased"""							)
								a									=				config.pop("""_commit_hash"""						, __magic_name__							)
								# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
								self.assertEqual(__magic_name__						, {"""do_lower_case""": False}							)

								# This model does not have a tokenizer_config so we get back an empty dict.
								a									=				get_tokenizer_config(__magic_name__							)
								self.assertDictEqual(__magic_name__						, {}							)

								# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
								a									=				AutoTokenizer.from_pretrained(__magic_name__							)
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				get_tokenizer_config(__magic_name__							)

								# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
								self.assertEqual(config["""tokenizer_class"""]						, """BertTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)

									a									=				CustomTokenizer.from_pretrained(__magic_name__							)
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							@require_tokenizers
							def  lowerCamelCase__				(		self		:Tuple							):







								'''simple docstring'''







								try:
									AutoConfig.register("""custom"""						, __magic_name__							)

									# Can register in two steps
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, None)							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									del TOKENIZER_MAPPING._extra_content[CustomConfig]
									# Can register in one step
									AutoTokenizer.register(
									    __magic_name__						, slow_tokenizer_class=__magic_name__						, fast_tokenizer_class=__magic_name__							)
									self.assertEqual(TOKENIZER_MAPPING[CustomConfig]						, (CustomTokenizer, CustomTokenizerFast)							)

									# Trying to register something existing in the Transformers library will raise an error
									with self.assertRaises(__magic_name__							):
										AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)

									# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
									# and that model does not have a tokenizer.json
									with tempfile.TemporaryDirectory() as tmp_dir:
										a									=				BertTokenizerFast.from_pretrained(__magic_name__							)
										bert_tokenizer.save_pretrained(__magic_name__							)
										a									=				CustomTokenizerFast.from_pretrained(__magic_name__							)

									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

										a									=				AutoTokenizer.from_pretrained(__magic_name__						, use_fast=__magic_name__							)
										self.assertIsInstance(__magic_name__						, __magic_name__							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:int							):







								'''simple docstring'''







								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
								# If remote code is disabled, we can't load this config.
								with self.assertRaises(__magic_name__							):
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)

								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								# Test tokenizer can be reloaded.
								with tempfile.TemporaryDirectory() as tmp_dir:
									tokenizer.save_pretrained(__magic_name__							)
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__							)
								self.assertTrue(reloaded_tokenizer.special_attribute_present							)

								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									# Test tokenizer can be reloaded.
									with tempfile.TemporaryDirectory() as tmp_dir:
										tokenizer.save_pretrained(__magic_name__							)
										a									=				AutoTokenizer.from_pretrained(__magic_name__						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(reloaded_tokenizer.special_attribute_present							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertEqual(reloaded_tokenizer.__class__.__name__						, """NewTokenizer"""							)
							@require_tokenizers
							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    False







								class 						__lowerCAmelCase      (      __magic_name__					):
															UpperCamelCase__   	=    NewTokenizer
															UpperCamelCase__   	=    False

								try:
									AutoConfig.register("""custom"""						, __magic_name__							)
									AutoTokenizer.register(__magic_name__						, slow_tokenizer_class=__magic_name__							)
									AutoTokenizer.register(__magic_name__						, fast_tokenizer_class=__magic_name__							)
									# If remote code is not set, the default is to use local
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote code is disabled, we load the local one.
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertFalse(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertFalse(tokenizer.special_attribute_present							)

									# If remote is enabled, we load from the Hub
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)
									self.assertTrue(tokenizer.special_attribute_present							)
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
									self.assertTrue(tokenizer.special_attribute_present							)

								finally:
									if "custom" in CONFIG_MAPPING._extra_content:
										del CONFIG_MAPPING._extra_content["custom"]
									if CustomConfig in TOKENIZER_MAPPING._extra_content:
										del TOKENIZER_MAPPING._extra_content[CustomConfig]
							def  lowerCamelCase__				(		self		:Any							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained(
								    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__							)
								self.assertTrue(tokenizer.special_attribute_present							)
								if is_tokenizers_available():
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizerFast"""							)

									# Test we can also load the slow version
									a									=				AutoTokenizer.from_pretrained(
									    """hf-internal-testing/test_dynamic_tokenizer_legacy"""						, trust_remote_code=__magic_name__						, use_fast=__magic_name__							)
									self.assertTrue(tokenizer.special_attribute_present							)
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
								else:
									self.assertEqual(tokenizer.__class__.__name__						, """NewTokenizer"""							)
							def  lowerCamelCase__				(		self		:Union[str, Any]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, """bert-base is not a local folder and is not a valid model identifier"""							):
									a									=				AutoTokenizer.from_pretrained("""bert-base"""							)
							def  lowerCamelCase__				(		self		:List[str]							):







								'''simple docstring'''







								with self.assertRaisesRegex(
								    __magic_name__						, r"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
									a									=				AutoTokenizer.from_pretrained(__magic_name__						, revision="""aaaaaa"""							)





							def  lowerCamelCase__				(		self		:str							):







								'''simple docstring'''







								a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
								with RequestCounter() as counter:
									a									=				AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
									self.assertEqual(counter.get_request_count						, 0							)
									self.assertEqual(counter.head_request_count						, 1							)
									self.assertEqual(counter.other_request_count						, 0							)

