"""simple docstring"""


def      __UpperCAmelCase       (      lowercase		):







  """simple docstring"""
  _UpperCAmelCase    			=   0
  _UpperCAmelCase    			=   len(lowercase		)

  for i in range(n - 1		):
    for j in range(i + 1 ,lowercase		):
      if arr[i] > arr[j]:
        num_inversions += 1

  return num_inversions






def      __UpperCAmelCase       (      lowercase		):







  """simple docstring"""
  if len(lowercase		) <= 1:
    return arr, 0
  _UpperCAmelCase    			=   len(lowercase		) // 2
  _UpperCAmelCase    			=   arr[0:mid]
  _UpperCAmelCase    			=   arr[mid:]

  _UpperCAmelCase			,					_UpperCAmelCase    			=   count_inversions_recursive(lowercase		)
  _UpperCAmelCase			,					_UpperCAmelCase    			=   count_inversions_recursive(lowercase		)
  _UpperCAmelCase			,					_UpperCAmelCase    			=   _count_cross_inversions(lowercase ,lowercase		)

  _UpperCAmelCase    			=   inversion_p + inversions_q + cross_inversions
  return c, num_inversions






def      __UpperCAmelCase       (      lowercase ,lowercase		):







  """simple docstring"""
  _UpperCAmelCase    			=   []
  _UpperCAmelCase    			=   _UpperCAmelCase    			=   _UpperCAmelCase    			=   0
  while i < len(lowercase		) and j < len(lowercase		):
    if p[i] > q[j]:
      # if P[1] > Q[j], then P[k] > Q[k] for all  i < k <= len(P)
      # These are all inversions. The claim emerges from the
      # property that P is sorted.
      num_inversion += len(lowercase		) - i
      r.append(q[j]		)
      j += 1
    else:
      r.append(p[i]		)
      i += 1

  if i < len(lowercase		):
    r.extend(p[i:]		)
  else:
    r.extend(q[j:]		)

  return r, num_inversion






def      __UpperCAmelCase       (      ):







  """simple docstring"""
  _UpperCAmelCase    			=   [10, 2, 1, 5, 5, 2, 11]

  # this arr has 8 inversions:
  # (10, 2), (10, 1), (10, 5), (10, 5), (10, 2), (2, 1), (5, 2), (5, 2)

  _UpperCAmelCase    			=   count_inversions_bf(lowercase		)
  _UpperCAmelCase			,					_UpperCAmelCase    			=   count_inversions_recursive(lowercase		)

  assert num_inversions_bf == num_inversions_recursive == 8

  print("""number of inversions = """ ,lowercase		)

  # testing an array with zero inversion (a sorted arr_1)

  arr_a.sort()
  _UpperCAmelCase    			=   count_inversions_bf(lowercase		)
  _UpperCAmelCase			,					_UpperCAmelCase    			=   count_inversions_recursive(lowercase		)

  assert num_inversions_bf == num_inversions_recursive == 0
  print("""number of inversions = """ ,lowercase		)

  # an empty list should also have zero inversions
  _UpperCAmelCase    			=   []
  _UpperCAmelCase    			=   count_inversions_bf(lowercase		)
  _UpperCAmelCase			,					_UpperCAmelCase    			=   count_inversions_recursive(lowercase		)

  assert num_inversions_bf == num_inversions_recursive == 0
  print("""number of inversions = """ ,lowercase		)


if __name__ == "__main__":
    main()

"""simple docstring"""


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast


class 							a					(		unittest.TestCase ):


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      _UpperCAmelCase    			=   0


    @slow
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )

      for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(GPTaTokenizer, GPTaTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )


    def   lowerCAmelCase_	(       self						:	Union[str, Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(RobertaTokenizer, RobertaTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					20       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      _UpperCAmelCase    			=   AutoConfig.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )
      # Check that tokenizer_type ≠ model_type
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					config=__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Tuple       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with pytest.raises(__lowerCAmelCase       ):
        AutoTokenizer.from_pretrained("""./""" ,					tokenizer_type="""xxx"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )

        if isinstance(__lowerCAmelCase ,					__lowerCAmelCase       ):
          self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,					__lowerCAmelCase       )
        else:
          self.assertEqual(tokenizer.do_lower_case ,					__lowerCAmelCase       )

        self.assertEqual(tokenizer.model_max_length ,					512       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        with self.assertRaisesRegex(
            __lowerCAmelCase ,					"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier""" ,					):
          _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""       )


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      # tests: https://github.com/huggingface/transformers/pull/13251
      # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
      # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
      _UpperCAmelCase    			=   TOKENIZER_MAPPING.values()
      _UpperCAmelCase    			=   []

      for slow_tok, fast_tok in tokenizers:
        if slow_tok is not None:
          tokenizer_names.append(slow_tok.__name__       )

        if fast_tok is not None:
          tokenizer_names.append(fast_tok.__name__       )

      for tokenizer_name in tokenizer_names:
        # must find the right class
        tokenizer_class_from_name(__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased""" ,					use_fast=__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""       ) ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	int       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""distilbert-base-uncased""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   """Hello, world. How are you?"""
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""microsoft/mpnet-base""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""       )
      self.assertEqual(type(__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertEqual(tokenizer.model_max_length ,					512       )
      self.assertEqual(tokenizer.vocab_size ,					3_0000       )
      self.assertEqual(tokenizer.unk_token ,					"""[UNK]"""       )
      self.assertEqual(tokenizer.padding_side ,					"""right"""       )
      self.assertEqual(tokenizer.truncation_side ,					"""right"""       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )

      self.assertIsInstance(__lowerCAmelCase ,					tokenizer.__class__       )
      self.assertEqual(tokenizera.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""ctrl"""       )
      # There is no fast CTRL so this always gives us a slow tokenizer.
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	str       ):
      # Check we can load the tokenizer config of an online model.
      _UpperCAmelCase    			=   get_tokenizer_config("""bert-base-cased"""       )
      _UpperCAmelCase    			=   config.pop("""_commit_hash""" ,					__lowerCAmelCase       )
      # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
      self.assertEqual(__lowerCAmelCase ,					{"""do_lower_case""": False}       )

      # This model does not have a tokenizer_config so we get back an empty dict.
      _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )
      self.assertDictEqual(__lowerCAmelCase ,					{}       )

      # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )

      # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
      self.assertEqual(config["""tokenizer_class"""] ,					"""BertTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )

        _UpperCAmelCase    			=   CustomTokenizer.from_pretrained(__lowerCAmelCase       )
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        # Can register in two steps
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, None)       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        del TOKENIZER_MAPPING._extra_content[CustomConfig]
        # Can register in one step
        AutoTokenizer.register(
            __lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )

        # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
        # and that model does not have a tokenizer.json
        with tempfile.TemporaryDirectory() as tmp_dir:
          _UpperCAmelCase    			=   BertTokenizerFast.from_pretrained(__lowerCAmelCase       )
          bert_tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   CustomTokenizerFast.from_pretrained(__lowerCAmelCase       )

        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # If remote code is not set, we will time out when asking whether to load the model.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      # Test tokenizer can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(reloaded_tokenizer.special_attribute_present       )

      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        # Test tokenizer can be reloaded.
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(reloaded_tokenizer.special_attribute_present       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):


      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Tuple  				=				False



      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Optional[int]  				=				NewTokenizer
          _snake_case  :	Any  				=				False

      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        # If remote code is not set, the default is to use local
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote code is disabled, we load the local one.
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote is enabled, we load from the Hub
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertTrue(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(tokenizer.special_attribute_present       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
          """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					"""bert-base is not a local folder and is not a valid model identifier"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""bert-base"""       )


    def   lowerCAmelCase_	(       self						:	int       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					revision="""aaaaaa"""       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # Make sure we have cached the tokenizer.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
      with RequestCounter() as counter:
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
        self.assertEqual(counter.get_request_count ,					0       )
        self.assertEqual(counter.head_request_count ,					1       )
        self.assertEqual(counter.other_request_count ,					0       )
