




import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / '''utils'''))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
	from test_module.custom_tokenization_fast import CustomTokenizerFast


class    SCREAMING_SNAKE_CASE_				(       unittest.TestCase   ):



	"""simple docstring"""


	def 					lowerCamelCase__     (					self		:						List[str]	)	->       Any:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      0






	@slow
	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Dict:







						"""simple docstring"""







						for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
											__UpperCamelCase					:	Optional[int]             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
											self.assertIsNotNone(lowerCAmelCase	)
											self.assertIsInstance(lowerCAmelCase					,     (BertTokenizer, BertTokenizerFast)	)
											self.assertGreater(len(lowerCAmelCase	)					,     0	)

						for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
											__UpperCamelCase					:	List[str]             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
											self.assertIsNotNone(lowerCAmelCase	)
											self.assertIsInstance(lowerCAmelCase					,     (GPTaTokenizer, GPTaTokenizerFast)	)
											self.assertGreater(len(lowerCAmelCase	)					,     0	)






	def 					lowerCamelCase__     (					self		:						int	)	->       Tuple:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
						self.assertIsInstance(lowerCAmelCase					,     (BertTokenizer, BertTokenizerFast)	)
						self.assertEqual(tokenizer.vocab_size					,     12	)






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       Union[str, Any]:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
						self.assertIsInstance(lowerCAmelCase					,     (RobertaTokenizer, RobertaTokenizerFast)	)
						self.assertEqual(tokenizer.vocab_size					,     20	)






	def 					lowerCamelCase__     (					self		:						Tuple	)	->       int:







						"""simple docstring"""







						__UpperCamelCase					:	List[str]             =      AutoConfig.from_pretrained(lowerCAmelCase	)
						self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)
						# Check that tokenizer_type ≠ model_type
						__UpperCamelCase					:	int             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     config=lowerCAmelCase	)
						self.assertIsInstance(lowerCAmelCase					,     (BertTokenizer, BertTokenizerFast)	)
						self.assertEqual(tokenizer.vocab_size					,     12	)






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       Union[str, Any]:







						"""simple docstring"""







						with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy("""./tests/fixtures/vocab.txt"""					,     os.path.join(lowerCAmelCase					,     """vocab.txt"""	)	)

											__UpperCamelCase					:	Optional[int]             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     tokenizer_type="""bert"""					,     use_fast=lowerCAmelCase	)
											self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)

						with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy("""./tests/fixtures/vocab.json"""					,     os.path.join(lowerCAmelCase					,     """vocab.json"""	)	)
											shutil.copy("""./tests/fixtures/merges.txt"""					,     os.path.join(lowerCAmelCase					,     """merges.txt"""	)	)

											__UpperCamelCase					:	Any             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     tokenizer_type="""gpt2"""					,     use_fast=lowerCAmelCase	)
											self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Any:







						"""simple docstring"""







						with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy("""./tests/fixtures/vocab.txt"""					,     os.path.join(lowerCAmelCase					,     """vocab.txt"""	)	)

											__UpperCamelCase					:	List[Any]             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     tokenizer_type="""bert"""	)
											self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)

						with tempfile.TemporaryDirectory() as tmp_dir:
											shutil.copy("""./tests/fixtures/vocab.json"""					,     os.path.join(lowerCAmelCase					,     """vocab.json"""	)	)
											shutil.copy("""./tests/fixtures/merges.txt"""					,     os.path.join(lowerCAmelCase					,     """merges.txt"""	)	)

											__UpperCamelCase					:	str             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     tokenizer_type="""gpt2"""	)
											self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						int	)	->       List[Any]:







						"""simple docstring"""







						with pytest.raises(lowerCAmelCase	):
											AutoTokenizer.from_pretrained("""./"""					,     tokenizer_type="""xxx"""	)






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       str:







						"""simple docstring"""







						for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
											__UpperCamelCase					:	str             =      tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""	)
											self.assertIsInstance(lowerCAmelCase					,     (BertTokenizer, BertTokenizerFast)	)

											if isinstance(lowerCAmelCase					,     lowerCAmelCase	):
																self.assertEqual(tokenizer.basic_tokenizer.do_lower_case					,     lowerCAmelCase	)
											else:
																self.assertEqual(tokenizer.do_lower_case					,     lowerCAmelCase	)

											self.assertEqual(tokenizer.model_max_length					,     512	)






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						List[Any]	)	->       List[str]:







						"""simple docstring"""







						for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
											with self.assertRaisesRegex(
											    lowerCAmelCase					,     """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""					,     ):
																__UpperCamelCase					:	Optional[Any]             =      tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""	)






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       str:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      TOKENIZER_MAPPING.values()
						__UpperCamelCase					:	int             =      []

						for slow_tok, fast_tok in tokenizers:
											if slow_tok is not None:
																tokenizer_names.append(slow_tok.__name__	)

											if fast_tok is not None:
																tokenizer_names.append(fast_tok.__name__	)

						for tokenizer_name in tokenizer_names:
											# must find the right class
											tokenizer_class_from_name(lowerCAmelCase	)






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						List[str]	)	->       int:







						"""simple docstring"""







						self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""					,     use_fast=lowerCAmelCase	)					,     lowerCAmelCase	)
						self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""	)					,     lowerCAmelCase	)






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						List[Any]	)	->       int:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      AutoTokenizer.from_pretrained("""distilbert-base-uncased"""					,     do_lower_case=lowerCAmelCase	)
						__UpperCamelCase					:	Union[str, Any]             =      "Hello, world. How are you?"
						__UpperCamelCase					:	int             =      tokenizer.tokenize(lowerCAmelCase	)
						self.assertEqual("""[UNK]"""					,     tokens[0]	)

						__UpperCamelCase					:	List[str]             =      AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""					,     do_lower_case=lowerCAmelCase	)
						__UpperCamelCase					:	Optional[int]             =      tokenizer.tokenize(lowerCAmelCase	)
						self.assertEqual("""[UNK]"""					,     tokens[0]	)






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						int	)	->       int:







						"""simple docstring"""







						__UpperCamelCase					:	Dict             =      AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""	)
						self.assertEqual(type(lowerCAmelCase	)					,     lowerCAmelCase	)
						self.assertEqual(tokenizer.model_max_length					,     512	)
						self.assertEqual(tokenizer.vocab_size					,     30000	)
						self.assertEqual(tokenizer.unk_token					,     """[UNK]"""	)
						self.assertEqual(tokenizer.padding_side					,     """right"""	)
						self.assertEqual(tokenizer.truncation_side					,     """right"""	)






	def 					lowerCamelCase__     (					self		:						Any	)	->       int:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[Any]             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
						self.assertIsInstance(lowerCAmelCase					,     (BertTokenizer, BertTokenizerFast)	)
						with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(lowerCAmelCase	)
											__UpperCamelCase					:	Optional[int]             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)

						self.assertIsInstance(lowerCAmelCase					,     tokenizer.__class__	)
						self.assertEqual(tokenizera.vocab_size					,     12	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       Optional[int]:







						"""simple docstring"""







						__UpperCamelCase					:	Dict             =      AutoTokenizer.from_pretrained("""ctrl"""	)
						# There is no fast CTRL so this always gives us a slow tokenizer.
						self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)






	def 					lowerCamelCase__     (					self		:						str	)	->       List[str]:







						"""simple docstring"""







						__UpperCamelCase					:	Union[str, Any]             =      get_tokenizer_config("""bert-base-cased"""	)
						__UpperCamelCase					:	Optional[int]             =      config.pop("""_commit_hash"""					,     lowerCAmelCase	)
						# If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
						self.assertEqual(lowerCAmelCase					,     {"""do_lower_case""": False}	)

						# This model does not have a tokenizer_config so we get back an empty dict.
						__UpperCamelCase					:	Optional[int]             =      get_tokenizer_config(lowerCAmelCase	)
						self.assertDictEqual(lowerCAmelCase					,     {}	)

						# A tokenizer saved with `save_pretrained` always creates a tokenizer config.
						__UpperCamelCase					:	Dict             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
						with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(lowerCAmelCase	)
											__UpperCamelCase					:	Optional[Any]             =      get_tokenizer_config(lowerCAmelCase	)

						# Check the class of the tokenizer was properly saved (note that it always saves the slow class).
						self.assertEqual(config["""tokenizer_class"""]					,     """BertTokenizer"""	)






	def 					lowerCamelCase__     (					self		:						Optional[int]	)	->       Optional[Any]:







						"""simple docstring"""







						try:
											AutoConfig.register("""custom"""					,     lowerCAmelCase	)

											AutoTokenizer.register(lowerCAmelCase					,     slow_tokenizer_class=lowerCAmelCase	)
											# Trying to register something existing in the Transformers library will raise an error
											with self.assertRaises(lowerCAmelCase	):
																AutoTokenizer.register(lowerCAmelCase					,     slow_tokenizer_class=lowerCAmelCase	)

											__UpperCamelCase					:	Optional[int]             =      CustomTokenizer.from_pretrained(lowerCAmelCase	)
											with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(lowerCAmelCase	)

																__UpperCamelCase					:	int             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
																self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)

						finally:
											if "custom" in CONFIG_MAPPING._extra_content:
																del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
																del TOKENIZER_MAPPING._extra_content[CustomConfig]






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						Any	)	->       Union[str, Any]:







						"""simple docstring"""







						try:
											AutoConfig.register("""custom"""					,     lowerCAmelCase	)

											# Can register in two steps
											AutoTokenizer.register(lowerCAmelCase					,     slow_tokenizer_class=lowerCAmelCase	)
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,     (CustomTokenizer, None)	)
											AutoTokenizer.register(lowerCAmelCase					,     fast_tokenizer_class=lowerCAmelCase	)
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,     (CustomTokenizer, CustomTokenizerFast)	)

											del TOKENIZER_MAPPING._extra_content[CustomConfig]
											# Can register in one step
											AutoTokenizer.register(
											    lowerCAmelCase					,     slow_tokenizer_class=lowerCAmelCase					,     fast_tokenizer_class=lowerCAmelCase	)
											self.assertEqual(TOKENIZER_MAPPING[CustomConfig]					,     (CustomTokenizer, CustomTokenizerFast)	)

											# Trying to register something existing in the Transformers library will raise an error
											with self.assertRaises(lowerCAmelCase	):
																AutoTokenizer.register(lowerCAmelCase					,     fast_tokenizer_class=lowerCAmelCase	)

											# We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
											# and that model does not have a tokenizer.json
											with tempfile.TemporaryDirectory() as tmp_dir:
																__UpperCamelCase					:	Any             =      BertTokenizerFast.from_pretrained(lowerCAmelCase	)
																bert_tokenizer.save_pretrained(lowerCAmelCase	)
																__UpperCamelCase					:	Tuple             =      CustomTokenizerFast.from_pretrained(lowerCAmelCase	)

											with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(lowerCAmelCase	)

																__UpperCamelCase					:	Optional[int]             =      AutoTokenizer.from_pretrained(lowerCAmelCase	)
																self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)

																__UpperCamelCase					:	str             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     use_fast=lowerCAmelCase	)
																self.assertIsInstance(lowerCAmelCase					,     lowerCAmelCase	)

						finally:
											if "custom" in CONFIG_MAPPING._extra_content:
																del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
																del TOKENIZER_MAPPING._extra_content[CustomConfig]






	def 					lowerCamelCase__     (					self		:						Tuple	)	->       Optional[int]:







						"""simple docstring"""







						with self.assertRaises(lowerCAmelCase	):
											__UpperCamelCase					:	List[Any]             =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""	)
						# If remote code is disabled, we can't load this config.
						with self.assertRaises(lowerCAmelCase	):
											__UpperCamelCase					:	Optional[int]             =      AutoTokenizer.from_pretrained(
											    """hf-internal-testing/test_dynamic_tokenizer"""					,     trust_remote_code=lowerCAmelCase	)

						__UpperCamelCase					:	int             =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					,     trust_remote_code=lowerCAmelCase	)
						self.assertTrue(tokenizer.special_attribute_present	)
						# Test tokenizer can be reloaded.
						with tempfile.TemporaryDirectory() as tmp_dir:
											tokenizer.save_pretrained(lowerCAmelCase	)
											__UpperCamelCase					:	List[Any]             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     trust_remote_code=lowerCAmelCase	)
						self.assertTrue(reloaded_tokenizer.special_attribute_present	)

						if is_tokenizers_available():
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizerFast"""	)
											self.assertEqual(reloaded_tokenizer.__class__.__name__					,     """NewTokenizerFast"""	)

											# Test we can also load the slow version
											__UpperCamelCase					:	List[str]             =      AutoTokenizer.from_pretrained(
											    """hf-internal-testing/test_dynamic_tokenizer"""					,     trust_remote_code=lowerCAmelCase					,     use_fast=lowerCAmelCase	)
											self.assertTrue(tokenizer.special_attribute_present	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizer"""	)
											# Test tokenizer can be reloaded.
											with tempfile.TemporaryDirectory() as tmp_dir:
																tokenizer.save_pretrained(lowerCAmelCase	)
																__UpperCamelCase					:	List[str]             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     trust_remote_code=lowerCAmelCase					,     use_fast=lowerCAmelCase	)
											self.assertEqual(reloaded_tokenizer.__class__.__name__					,     """NewTokenizer"""	)
											self.assertTrue(reloaded_tokenizer.special_attribute_present	)
						else:
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizer"""	)
											self.assertEqual(reloaded_tokenizer.__class__.__name__					,     """NewTokenizer"""	)






	@require_tokenizers
	def 					lowerCamelCase__     (					self		:						Optional[int]	)	->       int:







						"""simple docstring"""







						class    SCREAMING_SNAKE_CASE_				(       __lowercase   ):



							"""simple docstring"""


							__magic_name__   :       Tuple      	=		False






						class    SCREAMING_SNAKE_CASE_				(       __lowercase   ):



							"""simple docstring"""


							__magic_name__   :       Optional[Any]      	=		NewTokenizer
							__magic_name__   :       Any      	=		False

						try:
											AutoConfig.register("""custom"""					,     lowerCAmelCase	)
											AutoTokenizer.register(lowerCAmelCase					,     slow_tokenizer_class=lowerCAmelCase	)
											AutoTokenizer.register(lowerCAmelCase					,     fast_tokenizer_class=lowerCAmelCase	)
											# If remote code is not set, the default is to use local
											__UpperCamelCase					:	Dict             =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizerFast"""	)
											self.assertFalse(tokenizer.special_attribute_present	)
											__UpperCamelCase					:	Dict             =      AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""					,     use_fast=lowerCAmelCase	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizer"""	)
											self.assertFalse(tokenizer.special_attribute_present	)

											# If remote code is disabled, we load the local one.
											__UpperCamelCase					:	Dict             =      AutoTokenizer.from_pretrained(
											    """hf-internal-testing/test_dynamic_tokenizer"""					,     trust_remote_code=lowerCAmelCase	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizerFast"""	)
											self.assertFalse(tokenizer.special_attribute_present	)
											__UpperCamelCase					:	Dict             =      AutoTokenizer.from_pretrained(
											    """hf-internal-testing/test_dynamic_tokenizer"""					,     trust_remote_code=lowerCAmelCase					,     use_fast=lowerCAmelCase	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizer"""	)
											self.assertFalse(tokenizer.special_attribute_present	)

											# If remote is enabled, we load from the Hub
											__UpperCamelCase					:	Any             =      AutoTokenizer.from_pretrained(
											    """hf-internal-testing/test_dynamic_tokenizer"""					,     trust_remote_code=lowerCAmelCase	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizerFast"""	)
											self.assertTrue(tokenizer.special_attribute_present	)
											__UpperCamelCase					:	Union[str, Any]             =      AutoTokenizer.from_pretrained(
											    """hf-internal-testing/test_dynamic_tokenizer"""					,     trust_remote_code=lowerCAmelCase					,     use_fast=lowerCAmelCase	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizer"""	)
											self.assertTrue(tokenizer.special_attribute_present	)

						finally:
											if "custom" in CONFIG_MAPPING._extra_content:
																del CONFIG_MAPPING._extra_content["custom"]
											if CustomConfig in TOKENIZER_MAPPING._extra_content:
																del TOKENIZER_MAPPING._extra_content[CustomConfig]






	def 					lowerCamelCase__     (					self		:						Union[str, Any]	)	->       str:







						"""simple docstring"""







						__UpperCamelCase					:	Optional[int]             =      AutoTokenizer.from_pretrained(
						    """hf-internal-testing/test_dynamic_tokenizer_legacy"""					,     trust_remote_code=lowerCAmelCase	)
						self.assertTrue(tokenizer.special_attribute_present	)
						if is_tokenizers_available():
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizerFast"""	)

											# Test we can also load the slow version
											__UpperCamelCase					:	Optional[Any]             =      AutoTokenizer.from_pretrained(
											    """hf-internal-testing/test_dynamic_tokenizer_legacy"""					,     trust_remote_code=lowerCAmelCase					,     use_fast=lowerCAmelCase	)
											self.assertTrue(tokenizer.special_attribute_present	)
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizer"""	)
						else:
											self.assertEqual(tokenizer.__class__.__name__					,     """NewTokenizer"""	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       Any:







						"""simple docstring"""







						with self.assertRaisesRegex(
						    lowerCAmelCase					,     """bert-base is not a local folder and is not a valid model identifier"""	):
											__UpperCamelCase					:	str             =      AutoTokenizer.from_pretrained("""bert-base"""	)






	def 					lowerCamelCase__     (					self		:						Dict	)	->       Optional[int]:







						"""simple docstring"""







						with self.assertRaisesRegex(
						    lowerCAmelCase					,     R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""	):
											__UpperCamelCase					:	str             =      AutoTokenizer.from_pretrained(lowerCAmelCase					,     revision="""aaaaaa"""	)






	def 					lowerCamelCase__     (					self		:						List[str]	)	->       List[str]:







						"""simple docstring"""







						__UpperCamelCase					:	int             =      AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
						with RequestCounter() as counter:
											__UpperCamelCase					:	str             =      AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""	)
											self.assertEqual(counter.get_request_count					,     0	)
											self.assertEqual(counter.head_request_count					,     1	)
											self.assertEqual(counter.other_request_count					,     0	)








from .testing import (
    are_the_same_tensors,
    execute_subprocess_async,
    require_bnb,
    require_cpu,
    require_cuda,
    require_huggingface_suite,
    require_mps,
    require_multi_gpu,
    require_multi_xpu,
    require_safetensors,
    require_single_gpu,
    require_single_xpu,
    require_torch_min_version,
    require_tpu,
    require_xpu,
    skip,
    slow,
)
from .training import RegressionDataset, RegressionModel, RegressionModelaXPU


from .scripts import test_script, test_sync, test_ops  # isort: skip