





'''simple docstring'''

import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast
class 							__lowerCAmelCase	(       unittest.TestCase					):






       """simple docstring"""





       def snake_case__		(				self  :						str	)       ->	Optional[Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			0





       @slow
       def snake_case__		(				self  :						List[Any]	)       ->	int:

            '''simple docstring'''




            for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
                 self.assertIsNotNone(lowerCAmelCase__	)
                 self.assertIsInstance(lowerCAmelCase__			,					(BertTokenizer, BertTokenizerFast)	)
                 self.assertGreater(len(lowerCAmelCase__	)			,					0	)

            for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
                 self.assertIsNotNone(lowerCAmelCase__	)
                 self.assertIsInstance(lowerCAmelCase__			,					(GPTaTokenizer, GPTaTokenizerFast)	)
                 self.assertGreater(len(lowerCAmelCase__	)			,					0	)





       def snake_case__		(				self  :						str	)       ->	Union[str, Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
            self.assertIsInstance(lowerCAmelCase__			,					(BertTokenizer, BertTokenizerFast)	)
            self.assertEqual(tokenizer.vocab_size			,					12	)





       def snake_case__		(				self  :						Dict	)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
            self.assertIsInstance(lowerCAmelCase__			,					(RobertaTokenizer, RobertaTokenizerFast)	)
            self.assertEqual(tokenizer.vocab_size			,					20	)





       def snake_case__		(				self  :						Optional[int]	)       ->	Union[str, Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoConfig.from_pretrained(lowerCAmelCase__	)
            self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)
            # Check that tokenizer_type ≠ model_type
            _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					config=lowerCAmelCase__	)
            self.assertIsInstance(lowerCAmelCase__			,					(BertTokenizer, BertTokenizerFast)	)
            self.assertEqual(tokenizer.vocab_size			,					12	)





       def snake_case__		(				self  :						str	)       ->	List[Any]:

            '''simple docstring'''




            with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''			,					os.path.join(lowerCAmelCase__			,					'''vocab.txt'''	)	)

                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					tokenizer_type='''bert'''			,					use_fast=lowerCAmelCase__	)
                 self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)

            with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''			,					os.path.join(lowerCAmelCase__			,					'''vocab.json'''	)	)
                 shutil.copy('''./tests/fixtures/merges.txt'''			,					os.path.join(lowerCAmelCase__			,					'''merges.txt'''	)	)

                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					tokenizer_type='''gpt2'''			,					use_fast=lowerCAmelCase__	)
                 self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)





       @require_tokenizers
       def snake_case__		(				self  :						Optional[int]	)       ->	Tuple:

            '''simple docstring'''




            with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.txt'''			,					os.path.join(lowerCAmelCase__			,					'''vocab.txt'''	)	)

                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					tokenizer_type='''bert'''	)
                 self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)

            with tempfile.TemporaryDirectory() as tmp_dir:
                 shutil.copy('''./tests/fixtures/vocab.json'''			,					os.path.join(lowerCAmelCase__			,					'''vocab.json'''	)	)
                 shutil.copy('''./tests/fixtures/merges.txt'''			,					os.path.join(lowerCAmelCase__			,					'''merges.txt'''	)	)

                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					tokenizer_type='''gpt2'''	)
                 self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)





       def snake_case__		(				self  :						str	)       ->	Optional[Any]:

            '''simple docstring'''




            with pytest.raises(lowerCAmelCase__	):
                 AutoTokenizer.from_pretrained('''./'''			,					tokenizer_type='''xxx'''	)





       @require_tokenizers
       def snake_case__		(				self  :						Tuple	)       ->	Optional[int]:

            '''simple docstring'''




            for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 _UpperCamelCase							       =			tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''	)
                 self.assertIsInstance(lowerCAmelCase__			,					(BertTokenizer, BertTokenizerFast)	)

                 if isinstance(lowerCAmelCase__			,					lowerCAmelCase__	):
                      self.assertEqual(tokenizer.basic_tokenizer.do_lower_case			,					lowerCAmelCase__	)
                 else:
                      self.assertEqual(tokenizer.do_lower_case			,					lowerCAmelCase__	)

                 self.assertEqual(tokenizer.model_max_length			,					512	)





       @require_tokenizers
       def snake_case__		(				self  :						Dict	)       ->	Optional[Any]:

            '''simple docstring'''




            for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
                 with self.assertRaisesRegex(
                     lowerCAmelCase__			,					'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''			,					):
                      _UpperCamelCase							       =			tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''	)





       def snake_case__		(				self  :						Optional[Any]	)       ->	Union[str, Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			TOKENIZER_MAPPING.values()
            _UpperCamelCase							       =			[]

            for slow_tok, fast_tok in tokenizers:
                 if slow_tok is not None:
                      tokenizer_names.append(slow_tok.__name__	)

                 if fast_tok is not None:
                      tokenizer_names.append(fast_tok.__name__	)

            for tokenizer_name in tokenizer_names:
                 # must find the right class
                 tokenizer_class_from_name(lowerCAmelCase__	)





       @require_tokenizers
       def snake_case__		(				self  :						Tuple	)       ->	str:

            '''simple docstring'''




            self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''			,					use_fast=lowerCAmelCase__	)			,					lowerCAmelCase__	)
            self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''	)			,					lowerCAmelCase__	)





       @require_tokenizers
       def snake_case__		(				self  :						Optional[int]	)       ->	List[Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''distilbert-base-uncased'''			,					do_lower_case=lowerCAmelCase__	)
            _UpperCamelCase							       =			'''Hello, world. How are you?'''
            _UpperCamelCase							       =			tokenizer.tokenize(lowerCAmelCase__	)
            self.assertEqual('''[UNK]'''			,					tokens[0]	)

            _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''			,					do_lower_case=lowerCAmelCase__	)
            _UpperCamelCase							       =			tokenizer.tokenize(lowerCAmelCase__	)
            self.assertEqual('''[UNK]'''			,					tokens[0]	)





       @require_tokenizers
       def snake_case__		(				self  :						Union[str, Any]	)       ->	Optional[int]:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''	)
            self.assertEqual(type(lowerCAmelCase__	)			,					lowerCAmelCase__	)
            self.assertEqual(tokenizer.model_max_length			,					512	)
            self.assertEqual(tokenizer.vocab_size			,					30000	)
            self.assertEqual(tokenizer.unk_token			,					'''[UNK]'''	)
            self.assertEqual(tokenizer.padding_side			,					'''right'''	)
            self.assertEqual(tokenizer.truncation_side			,					'''right'''	)





       def snake_case__		(				self  :						Optional[int]	)       ->	Tuple:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
            self.assertIsInstance(lowerCAmelCase__			,					(BertTokenizer, BertTokenizerFast)	)
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(lowerCAmelCase__	)
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)

            self.assertIsInstance(lowerCAmelCase__			,					tokenizer.__class__	)
            self.assertEqual(tokenizera.vocab_size			,					12	)





       def snake_case__		(				self  :						str	)       ->	List[str]:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''ctrl'''	)
            # There is no fast CTRL so this always gives us a slow tokenizer.
            self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)





       def snake_case__		(				self  :						Optional[int]	)       ->	str:

            '''simple docstring'''




            _UpperCamelCase							       =			get_tokenizer_config('''bert-base-cased'''	)
            _UpperCamelCase							       =			config.pop('''_commit_hash'''			,					lowerCAmelCase__	)
            # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
            self.assertEqual(lowerCAmelCase__			,					{'''do_lower_case''': False}	)

            # This model does not have a tokenizer_config so we get back an empty dict.
            _UpperCamelCase							       =			get_tokenizer_config(lowerCAmelCase__	)
            self.assertDictEqual(lowerCAmelCase__			,					{}	)

            # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
            _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(lowerCAmelCase__	)
                 _UpperCamelCase							       =			get_tokenizer_config(lowerCAmelCase__	)

            # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
            self.assertEqual(config['''tokenizer_class''']			,					'''BertTokenizer'''	)





       def snake_case__		(				self  :						Union[str, Any]	)       ->	List[str]:

            '''simple docstring'''




            try:
                 AutoConfig.register('''custom'''			,					lowerCAmelCase__	)

                 AutoTokenizer.register(lowerCAmelCase__			,					slow_tokenizer_class=lowerCAmelCase__	)
                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(lowerCAmelCase__	):
                      AutoTokenizer.register(lowerCAmelCase__			,					slow_tokenizer_class=lowerCAmelCase__	)

                 _UpperCamelCase							       =			CustomTokenizer.from_pretrained(lowerCAmelCase__	)
                 with tempfile.TemporaryDirectory() as tmp_dir:
                      tokenizer.save_pretrained(lowerCAmelCase__	)

                      _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
                      self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)

            finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                      del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                      del TOKENIZER_MAPPING._extra_content[CustomConfig]





       @require_tokenizers
       def snake_case__		(				self  :						Tuple	)       ->	str:

            '''simple docstring'''




            try:
                 AutoConfig.register('''custom'''			,					lowerCAmelCase__	)

                 # Can register in two steps
                 AutoTokenizer.register(lowerCAmelCase__			,					slow_tokenizer_class=lowerCAmelCase__	)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,					(CustomTokenizer, None)	)
                 AutoTokenizer.register(lowerCAmelCase__			,					fast_tokenizer_class=lowerCAmelCase__	)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,					(CustomTokenizer, CustomTokenizerFast)	)

                 del TOKENIZER_MAPPING._extra_content[CustomConfig]
                 # Can register in one step
                 AutoTokenizer.register(
                     lowerCAmelCase__			,					slow_tokenizer_class=lowerCAmelCase__			,					fast_tokenizer_class=lowerCAmelCase__	)
                 self.assertEqual(TOKENIZER_MAPPING[CustomConfig]			,					(CustomTokenizer, CustomTokenizerFast)	)

                 # Trying to register something existing in the Transformers library will raise an error
                 with self.assertRaises(lowerCAmelCase__	):
                      AutoTokenizer.register(lowerCAmelCase__			,					fast_tokenizer_class=lowerCAmelCase__	)

                 # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
                 # and that model does not have a tokenizer.json
                 with tempfile.TemporaryDirectory() as tmp_dir:
                      _UpperCamelCase							       =			BertTokenizerFast.from_pretrained(lowerCAmelCase__	)
                      bert_tokenizer.save_pretrained(lowerCAmelCase__	)
                      _UpperCamelCase							       =			CustomTokenizerFast.from_pretrained(lowerCAmelCase__	)

                 with tempfile.TemporaryDirectory() as tmp_dir:
                      tokenizer.save_pretrained(lowerCAmelCase__	)

                      _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__	)
                      self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)

                      _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					use_fast=lowerCAmelCase__	)
                      self.assertIsInstance(lowerCAmelCase__			,					lowerCAmelCase__	)

            finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                      del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                      del TOKENIZER_MAPPING._extra_content[CustomConfig]





       def snake_case__		(				self  :						Union[str, Any]	)       ->	Union[str, Any]:

            '''simple docstring'''




            with self.assertRaises(lowerCAmelCase__	):
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''	)
            # If remote code is disabled, we can't load this config.
            with self.assertRaises(lowerCAmelCase__	):
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''			,					trust_remote_code=lowerCAmelCase__	)

            _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''			,					trust_remote_code=lowerCAmelCase__	)
            self.assertTrue(tokenizer.special_attribute_present	)
            # Test tokenizer can be reloaded.
            with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer.save_pretrained(lowerCAmelCase__	)
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					trust_remote_code=lowerCAmelCase__	)
            self.assertTrue(reloaded_tokenizer.special_attribute_present	)

            if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizerFast'''	)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__			,					'''NewTokenizerFast'''	)

                 # Test we can also load the slow version
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''			,					trust_remote_code=lowerCAmelCase__			,					use_fast=lowerCAmelCase__	)
                 self.assertTrue(tokenizer.special_attribute_present	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizer'''	)
                 # Test tokenizer can be reloaded.
                 with tempfile.TemporaryDirectory() as tmp_dir:
                      tokenizer.save_pretrained(lowerCAmelCase__	)
                      _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					trust_remote_code=lowerCAmelCase__			,					use_fast=lowerCAmelCase__	)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__			,					'''NewTokenizer'''	)
                 self.assertTrue(reloaded_tokenizer.special_attribute_present	)
            else:
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizer'''	)
                 self.assertEqual(reloaded_tokenizer.__class__.__name__			,					'''NewTokenizer'''	)





       @require_tokenizers
       def snake_case__		(				self  :						Optional[Any]	)       ->	Dict:

            '''simple docstring'''




            class 							__lowerCAmelCase	(       __magic_name__					):






                   """simple docstring"""
                   _snake_case      :	int									=       False

            class 							__lowerCAmelCase	(       __magic_name__					):






                   """simple docstring"""
                   _snake_case      :	List[Any]									=       NewTokenizer
                   _snake_case      :	str									=       False

            try:
                 AutoConfig.register('''custom'''			,					lowerCAmelCase__	)
                 AutoTokenizer.register(lowerCAmelCase__			,					slow_tokenizer_class=lowerCAmelCase__	)
                 AutoTokenizer.register(lowerCAmelCase__			,					fast_tokenizer_class=lowerCAmelCase__	)
                 # If remote code is not set, the default is to use local
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizerFast'''	)
                 self.assertFalse(tokenizer.special_attribute_present	)
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''			,					use_fast=lowerCAmelCase__	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizer'''	)
                 self.assertFalse(tokenizer.special_attribute_present	)

                 # If remote code is disabled, we load the local one.
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''			,					trust_remote_code=lowerCAmelCase__	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizerFast'''	)
                 self.assertFalse(tokenizer.special_attribute_present	)
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''			,					trust_remote_code=lowerCAmelCase__			,					use_fast=lowerCAmelCase__	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizer'''	)
                 self.assertFalse(tokenizer.special_attribute_present	)

                 # If remote is enabled, we load from the Hub
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''			,					trust_remote_code=lowerCAmelCase__	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizerFast'''	)
                 self.assertTrue(tokenizer.special_attribute_present	)
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer'''			,					trust_remote_code=lowerCAmelCase__			,					use_fast=lowerCAmelCase__	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizer'''	)
                 self.assertTrue(tokenizer.special_attribute_present	)

            finally:
                 if "custom" in CONFIG_MAPPING._extra_content:
                      del CONFIG_MAPPING._extra_content["custom"]
                 if CustomConfig in TOKENIZER_MAPPING._extra_content:
                      del TOKENIZER_MAPPING._extra_content[CustomConfig]





       def snake_case__		(				self  :						Union[str, Any]	)       ->	Optional[Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                '''hf-internal-testing/test_dynamic_tokenizer_legacy'''			,					trust_remote_code=lowerCAmelCase__	)
            self.assertTrue(tokenizer.special_attribute_present	)
            if is_tokenizers_available():
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizerFast'''	)

                 # Test we can also load the slow version
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(
                     '''hf-internal-testing/test_dynamic_tokenizer_legacy'''			,					trust_remote_code=lowerCAmelCase__			,					use_fast=lowerCAmelCase__	)
                 self.assertTrue(tokenizer.special_attribute_present	)
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizer'''	)
            else:
                 self.assertEqual(tokenizer.__class__.__name__			,					'''NewTokenizer'''	)





       def snake_case__		(				self  :						Dict	)       ->	List[Any]:

            '''simple docstring'''




            with self.assertRaisesRegex(
                lowerCAmelCase__			,					'''bert-base is not a local folder and is not a valid model identifier'''	):
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''bert-base'''	)





       def snake_case__		(				self  :						Optional[int]	)       ->	int:

            '''simple docstring'''




            with self.assertRaisesRegex(
                lowerCAmelCase__			,					r'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''	):
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained(lowerCAmelCase__			,					revision='''aaaaaa'''	)







       def snake_case__		(				self  :						List[str]	)       ->	Any:

            '''simple docstring'''




            _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''	)
            with RequestCounter() as counter:
                 _UpperCamelCase							       =			AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''	)
                 self.assertEqual(counter.get_request_count			,					0	)
                 self.assertEqual(counter.head_request_count			,					1	)
                 self.assertEqual(counter.other_request_count			,					0	)










'''simple docstring'''

from datasets.utils.patching import _PatchedModuleObj, patch_submodule

from . import _test_patching







def    a__							(							)						->						Tuple:





     """simple docstring"""

     import os as original_os
     from os import path as original_path
     from os import rename as original_rename
     from os.path import dirname as original_dirname
     from os.path import join as original_join

     assert _test_patching.os is original_os
     assert _test_patching.path is original_path
     assert _test_patching.join is original_join

     assert _test_patching.renamed_os is original_os
     assert _test_patching.renamed_path is original_path
     assert _test_patching.renamed_join is original_join

     _UpperCamelCase							       =			'''__test_patch_submodule_mock__'''
     with patch_submodule(_test_patching,   '''os.path.join''',   lowercase ):
          # Every way to access os.path.join must be patched, and the rest must stay untouched

          # check os.path.join
          assert isinstance(_test_patching.os,   _PatchedModuleObj )
          assert isinstance(_test_patching.os.path,   _PatchedModuleObj )
          assert _test_patching.os.path.join is mock

          # check path.join
          assert isinstance(_test_patching.path,   _PatchedModuleObj )
          assert _test_patching.path.join is mock

          # check join
          assert _test_patching.join is mock

          # check that the other attributes are untouched
          assert _test_patching.os.rename is original_rename
          assert _test_patching.path.dirname is original_dirname
          assert _test_patching.os.path.dirname is original_dirname

          # Even renamed modules or objects must be patched

          # check renamed_os.path.join
          assert isinstance(_test_patching.renamed_os,   _PatchedModuleObj )
          assert isinstance(_test_patching.renamed_os.path,   _PatchedModuleObj )
          assert _test_patching.renamed_os.path.join is mock

          # check renamed_path.join
          assert isinstance(_test_patching.renamed_path,   _PatchedModuleObj )
          assert _test_patching.renamed_path.join is mock

          # check renamed_join
          assert _test_patching.renamed_join is mock

          # check that the other attributes are untouched
          assert _test_patching.renamed_os.rename is original_rename
          assert _test_patching.renamed_path.dirname is original_dirname
          assert _test_patching.renamed_os.path.dirname is original_dirname

     # check that everthing is back to normal when the patch is over

     assert _test_patching.os is original_os
     assert _test_patching.path is original_path
     assert _test_patching.join is original_join

     assert _test_patching.renamed_os is original_os
     assert _test_patching.renamed_path is original_path
     assert _test_patching.renamed_join is original_join







def    a__							(							)						->						Tuple:





     """simple docstring"""

     assert _test_patching.open is open

     _UpperCamelCase							       =			'''__test_patch_submodule_builtin_mock__'''
     # _test_patching has "open" in its globals
     assert _test_patching.open is open
     with patch_submodule(_test_patching,   '''open''',   lowercase ):
          assert _test_patching.open is mock

     # check that everthing is back to normal when the patch is over

     assert _test_patching.open is open







def    a__							(							)						->						Optional[Any]:





     """simple docstring"""

     _UpperCamelCase							       =			'''__test_patch_submodule_missing_mock__'''
     with patch_submodule(_test_patching,   '''pandas.read_csv''',   lowercase ):
          pass







def    a__							(							)						->						Optional[int]:





     """simple docstring"""

     _UpperCamelCase							       =			'''__test_patch_submodule_missing_builtin_mock__'''
     # _test_patching doesn't have "len" in its globals
     assert getattr(_test_patching,   '''len''',   lowercase ) is None
     with patch_submodule(_test_patching,   '''len''',   lowercase ):
          assert _test_patching.len is mock
     assert _test_patching.len is len







def    a__							(							)						->						Optional[Any]:





     """simple docstring"""

     _UpperCamelCase							       =			'''__test_patch_submodule_start_and_stop_mock__'''
     _UpperCamelCase							       =			patch_submodule(_test_patching,   '''open''',   lowercase )
     assert _test_patching.open is open
     patch.start()
     assert _test_patching.open is mock
     patch.stop()
     assert _test_patching.open is open







def    a__							(							)						->						List[str]:





     """simple docstring"""

     from os import rename as original_rename
     from os.path import dirname as original_dirname
     from os.path import join as original_join

     _UpperCamelCase							       =			'''__test_patch_submodule_successive_join__'''
     _UpperCamelCase							       =			'''__test_patch_submodule_successive_dirname__'''
     _UpperCamelCase							       =			'''__test_patch_submodule_successive_rename__'''
     assert _test_patching.os.path.join is original_join
     assert _test_patching.os.path.dirname is original_dirname
     assert _test_patching.os.rename is original_rename

     with patch_submodule(_test_patching,   '''os.path.join''',   lowercase ):
          with patch_submodule(_test_patching,   '''os.rename''',   lowercase ):
               with patch_submodule(_test_patching,   '''os.path.dirname''',   lowercase ):
                    assert _test_patching.os.path.join is mock_join
                    assert _test_patching.os.path.dirname is mock_dirname
                    assert _test_patching.os.rename is mock_rename

    # try another order
     with patch_submodule(_test_patching,   '''os.rename''',   lowercase ):
          with patch_submodule(_test_patching,   '''os.path.join''',   lowercase ):
               with patch_submodule(_test_patching,   '''os.path.dirname''',   lowercase ):
                    assert _test_patching.os.path.join is mock_join
                    assert _test_patching.os.path.dirname is mock_dirname
                    assert _test_patching.os.rename is mock_rename

     assert _test_patching.os.path.join is original_join
     assert _test_patching.os.path.dirname is original_dirname
     assert _test_patching.os.rename is original_rename







def    a__							(							)						->						str:





     """simple docstring"""

     _UpperCamelCase							       =			'''__test_patch_submodule_doesnt_exist_mock__'''
     with patch_submodule(_test_patching,   '''__module_that_doesn_exist__.__attribute_that_doesn_exist__''',   lowercase ):
          pass
     with patch_submodule(_test_patching,   '''os.__attribute_that_doesn_exist__''',   lowercase ):
          pass



