


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
      from test_module.custom_tokenization_fast import CustomTokenizerFast



class 	lowerCamelCase__			(      unittest.TestCase      ):




   def 							lowerCamelCase_   (   self						:						str							):




         '''simple docstring'''







         lowerCamelCase__:						Tuple							   = 0




   @slow
   def 							lowerCamelCase_   (   self						:						Optional[int]							):




         '''simple docstring'''







         for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
               lowerCamelCase__:						Optional[Any]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE							)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    (BertTokenizer, BertTokenizerFast)							)
               self.assertGreater(len(_SCREAMING_SNAKE_CASE							)    ,    0							)

         for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
               lowerCamelCase__:						Union[str, Any]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
               self.assertIsNotNone(_SCREAMING_SNAKE_CASE							)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    (GPTaTokenizer, GPTaTokenizerFast)							)
               self.assertGreater(len(_SCREAMING_SNAKE_CASE							)    ,    0							)




   def 							lowerCamelCase_   (   self						:						str							):




         '''simple docstring'''







         lowerCamelCase__:						List[str]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
         self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    (BertTokenizer, BertTokenizerFast)							)
         self.assertEqual(tokenizer.vocab_size    ,    12							)




   def 							lowerCamelCase_   (   self						:						Dict							):




         '''simple docstring'''







         lowerCamelCase__:						Any							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
         self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    (RobertaTokenizer, RobertaTokenizerFast)							)
         self.assertEqual(tokenizer.vocab_size    ,    20							)




   def 							lowerCamelCase_   (   self						:						Any							):




         '''simple docstring'''







         lowerCamelCase__:						int							   = AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE							)
         self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)
         # Check that tokenizer_type ≠ model_type
         lowerCamelCase__:						Optional[int]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    config=_SCREAMING_SNAKE_CASE							)
         self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    (BertTokenizer, BertTokenizerFast)							)
         self.assertEqual(tokenizer.vocab_size    ,    12							)




   def 							lowerCamelCase_   (   self						:						Optional[int]							):




         '''simple docstring'''







         with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("""./tests/fixtures/vocab.txt"""    ,    os.path.join(_SCREAMING_SNAKE_CASE    ,    """vocab.txt"""							)							)

               lowerCamelCase__:						Optional[int]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    tokenizer_type="""bert"""    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)

         with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("""./tests/fixtures/vocab.json"""    ,    os.path.join(_SCREAMING_SNAKE_CASE    ,    """vocab.json"""							)							)
               shutil.copy("""./tests/fixtures/merges.txt"""    ,    os.path.join(_SCREAMING_SNAKE_CASE    ,    """merges.txt"""							)							)

               lowerCamelCase__:						Optional[Any]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    tokenizer_type="""gpt2"""    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						List[str]							):




         '''simple docstring'''







         with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("""./tests/fixtures/vocab.txt"""    ,    os.path.join(_SCREAMING_SNAKE_CASE    ,    """vocab.txt"""							)							)

               lowerCamelCase__:						Tuple							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    tokenizer_type="""bert"""							)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)

         with tempfile.TemporaryDirectory() as tmp_dir:
               shutil.copy("""./tests/fixtures/vocab.json"""    ,    os.path.join(_SCREAMING_SNAKE_CASE    ,    """vocab.json"""							)							)
               shutil.copy("""./tests/fixtures/merges.txt"""    ,    os.path.join(_SCREAMING_SNAKE_CASE    ,    """merges.txt"""							)							)

               lowerCamelCase__:						Union[str, Any]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    tokenizer_type="""gpt2"""							)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)




   def 							lowerCamelCase_   (   self						:						Optional[int]							):




         '''simple docstring'''







         with pytest.raises(_SCREAMING_SNAKE_CASE							):
               AutoTokenizer.from_pretrained("""./"""    ,    tokenizer_type="""xxx"""							)




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						Tuple							):




         '''simple docstring'''







         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               lowerCamelCase__:						Any							   = tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""							)
               self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    (BertTokenizer, BertTokenizerFast)							)

               if isinstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							):
                     self.assertEqual(tokenizer.basic_tokenizer.do_lower_case    ,    _SCREAMING_SNAKE_CASE							)
               else:
                     self.assertEqual(tokenizer.do_lower_case    ,    _SCREAMING_SNAKE_CASE							)

               self.assertEqual(tokenizer.model_max_length    ,    512							)




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						Any							):




         '''simple docstring'''







         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
               with self.assertRaisesRegex(
                   _SCREAMING_SNAKE_CASE    ,    """julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier"""    ,    ):
                     lowerCamelCase__:						int							   = tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""							)




   def 							lowerCamelCase_   (   self						:						Dict							):




         '''simple docstring'''







         lowerCamelCase__:						int							   = TOKENIZER_MAPPING.values()
         lowerCamelCase__:						List[Any]							   = []

         for slow_tok, fast_tok in tokenizers:
               if slow_tok is not None:
                     tokenizer_names.append(slow_tok.__name__							)

               if fast_tok is not None:
                     tokenizer_names.append(fast_tok.__name__							)

         for tokenizer_name in tokenizer_names:
               # must find the right class
               tokenizer_class_from_name(_SCREAMING_SNAKE_CASE							)




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						Optional[int]							):




         '''simple docstring'''







         self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""    ,    use_fast=_SCREAMING_SNAKE_CASE							)    ,    _SCREAMING_SNAKE_CASE							)
         self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""							)    ,    _SCREAMING_SNAKE_CASE							)




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						int							):




         '''simple docstring'''







         lowerCamelCase__:						str							   = AutoTokenizer.from_pretrained("""distilbert-base-uncased"""    ,    do_lower_case=_SCREAMING_SNAKE_CASE							)
         lowerCamelCase__:						int							   = '''Hello, world. How are you?'''
         lowerCamelCase__:						List[Any]							   = tokenizer.tokenize(_SCREAMING_SNAKE_CASE							)
         self.assertEqual("""[UNK]"""    ,    tokens[0]							)

         lowerCamelCase__:						Any							   = AutoTokenizer.from_pretrained("""microsoft/mpnet-base"""    ,    do_lower_case=_SCREAMING_SNAKE_CASE							)
         lowerCamelCase__:						Tuple							   = tokenizer.tokenize(_SCREAMING_SNAKE_CASE							)
         self.assertEqual("""[UNK]"""    ,    tokens[0]							)




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						Optional[int]							):




         '''simple docstring'''







         lowerCamelCase__:						List[Any]							   = AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""							)
         self.assertEqual(type(_SCREAMING_SNAKE_CASE							)    ,    _SCREAMING_SNAKE_CASE							)
         self.assertEqual(tokenizer.model_max_length    ,    512							)
         self.assertEqual(tokenizer.vocab_size    ,    30000							)
         self.assertEqual(tokenizer.unk_token    ,    """[UNK]"""							)
         self.assertEqual(tokenizer.padding_side    ,    """right"""							)
         self.assertEqual(tokenizer.truncation_side    ,    """right"""							)




   def 							lowerCamelCase_   (   self						:						Optional[int]							):




         '''simple docstring'''







         lowerCamelCase__:						Any							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
         self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    (BertTokenizer, BertTokenizerFast)							)
         with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)
               lowerCamelCase__:						Dict							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)

         self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    tokenizer.__class__							)
         self.assertEqual(tokenizera.vocab_size    ,    12							)




   def 							lowerCamelCase_   (   self						:						int							):




         '''simple docstring'''







         lowerCamelCase__:						Tuple							   = AutoTokenizer.from_pretrained("""ctrl"""							)
         # There is no fast CTRL so this always gives us a slow tokenizer.
         self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)




   def 							lowerCamelCase_   (   self						:						Any							):




         '''simple docstring'''







         lowerCamelCase__:						int							   = get_tokenizer_config("""bert-base-cased"""							)
         lowerCamelCase__:						Optional[int]							   = config.pop("""_commit_hash"""    ,    _SCREAMING_SNAKE_CASE							)
         # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
         self.assertEqual(_SCREAMING_SNAKE_CASE    ,    {"""do_lower_case""": False}							)

         # This model does not have a tokenizer_config so we get back an empty dict.
         lowerCamelCase__:						Any							   = get_tokenizer_config(_SCREAMING_SNAKE_CASE							)
         self.assertDictEqual(_SCREAMING_SNAKE_CASE    ,    {}							)

         # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
         lowerCamelCase__:						Dict							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
         with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)
               lowerCamelCase__:						Dict							   = get_tokenizer_config(_SCREAMING_SNAKE_CASE							)

         # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
         self.assertEqual(config["""tokenizer_class"""]    ,    """BertTokenizer"""							)




   def 							lowerCamelCase_   (   self						:						Optional[Any]							):




         '''simple docstring'''







         try:
               AutoConfig.register("""custom"""    ,    _SCREAMING_SNAKE_CASE							)

               AutoTokenizer.register(_SCREAMING_SNAKE_CASE    ,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE							)
               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_SCREAMING_SNAKE_CASE							):
                     AutoTokenizer.register(_SCREAMING_SNAKE_CASE    ,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE							)

               lowerCamelCase__:						str							   = CustomTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
               with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)

                     lowerCamelCase__:						Dict							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
                     self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)

         finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                     del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                     del TOKENIZER_MAPPING._extra_content[CustomConfig]




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						Dict							):




         '''simple docstring'''







         try:
               AutoConfig.register("""custom"""    ,    _SCREAMING_SNAKE_CASE							)

               # Can register in two steps
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE    ,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,    (CustomTokenizer, None)							)
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE    ,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,    (CustomTokenizer, CustomTokenizerFast)							)

               del TOKENIZER_MAPPING._extra_content[CustomConfig]
               # Can register in one step
               AutoTokenizer.register(
                   _SCREAMING_SNAKE_CASE    ,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE    ,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(TOKENIZER_MAPPING[CustomConfig]    ,    (CustomTokenizer, CustomTokenizerFast)							)

               # Trying to register something existing in the Transformers library will raise an error
               with self.assertRaises(_SCREAMING_SNAKE_CASE							):
                     AutoTokenizer.register(_SCREAMING_SNAKE_CASE    ,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE							)

               # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
               # and that model does not have a tokenizer.json
               with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase__:						List[str]							   = BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE							)
                     bert_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)
                     lowerCamelCase__:						str							   = CustomTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE							)

               with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)

                     lowerCamelCase__:						Optional[int]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE							)
                     self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)

                     lowerCamelCase__:						Any							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    use_fast=_SCREAMING_SNAKE_CASE							)
                     self.assertIsInstance(_SCREAMING_SNAKE_CASE    ,    _SCREAMING_SNAKE_CASE							)

         finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                     del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                     del TOKENIZER_MAPPING._extra_content[CustomConfig]




   def 							lowerCamelCase_   (   self						:						List[str]							):




         '''simple docstring'''







         with self.assertRaises(_SCREAMING_SNAKE_CASE							):
               lowerCamelCase__:						Tuple							   = AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
         # If remote code is disabled, we can't load this config.
         with self.assertRaises(_SCREAMING_SNAKE_CASE							):
               lowerCamelCase__:						Union[str, Any]							   = AutoTokenizer.from_pretrained(
                   """hf-internal-testing/test_dynamic_tokenizer"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE							)

         lowerCamelCase__:						Dict							   = AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE							)
         self.assertTrue(tokenizer.special_attribute_present							)
         # Test tokenizer can be reloaded.
         with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)
               lowerCamelCase__:						int							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    trust_remote_code=_SCREAMING_SNAKE_CASE							)
         self.assertTrue(reloaded_tokenizer.special_attribute_present							)

         if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizerFast"""							)
               self.assertEqual(reloaded_tokenizer.__class__.__name__    ,    """NewTokenizerFast"""							)

               # Test we can also load the slow version
               lowerCamelCase__:						List[str]							   = AutoTokenizer.from_pretrained(
                   """hf-internal-testing/test_dynamic_tokenizer"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertTrue(tokenizer.special_attribute_present							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizer"""							)
               # Test tokenizer can be reloaded.
               with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE							)
                     lowerCamelCase__:						Optional[int]							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    trust_remote_code=_SCREAMING_SNAKE_CASE    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(reloaded_tokenizer.__class__.__name__    ,    """NewTokenizer"""							)
               self.assertTrue(reloaded_tokenizer.special_attribute_present							)
         else:
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizer"""							)
               self.assertEqual(reloaded_tokenizer.__class__.__name__    ,    """NewTokenizer"""							)




   @require_tokenizers
   def 							lowerCamelCase_   (   self						:						Union[str, Any]							):




         '''simple docstring'''







         class 	lowerCamelCase__			(      _a      ):
            __lowerCamelCase			      =  False



         class 	lowerCamelCase__			(      _a      ):
            __lowerCamelCase			      =  NewTokenizer
            __lowerCamelCase			      =  False

         try:
               AutoConfig.register("""custom"""    ,    _SCREAMING_SNAKE_CASE							)
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE    ,    slow_tokenizer_class=_SCREAMING_SNAKE_CASE							)
               AutoTokenizer.register(_SCREAMING_SNAKE_CASE    ,    fast_tokenizer_class=_SCREAMING_SNAKE_CASE							)
               # If remote code is not set, the default is to use local
               lowerCamelCase__:						int							   = AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizerFast"""							)
               self.assertFalse(tokenizer.special_attribute_present							)
               lowerCamelCase__:						Any							   = AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizer"""							)
               self.assertFalse(tokenizer.special_attribute_present							)

               # If remote code is disabled, we load the local one.
               lowerCamelCase__:						Tuple							   = AutoTokenizer.from_pretrained(
                   """hf-internal-testing/test_dynamic_tokenizer"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizerFast"""							)
               self.assertFalse(tokenizer.special_attribute_present							)
               lowerCamelCase__:						List[str]							   = AutoTokenizer.from_pretrained(
                   """hf-internal-testing/test_dynamic_tokenizer"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizer"""							)
               self.assertFalse(tokenizer.special_attribute_present							)

               # If remote is enabled, we load from the Hub
               lowerCamelCase__:						Optional[int]							   = AutoTokenizer.from_pretrained(
                   """hf-internal-testing/test_dynamic_tokenizer"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizerFast"""							)
               self.assertTrue(tokenizer.special_attribute_present							)
               lowerCamelCase__:						Any							   = AutoTokenizer.from_pretrained(
                   """hf-internal-testing/test_dynamic_tokenizer"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizer"""							)
               self.assertTrue(tokenizer.special_attribute_present							)

         finally:
               if "custom" in CONFIG_MAPPING._extra_content:
                     del CONFIG_MAPPING._extra_content["custom"]
               if CustomConfig in TOKENIZER_MAPPING._extra_content:
                     del TOKENIZER_MAPPING._extra_content[CustomConfig]




   def 							lowerCamelCase_   (   self						:						str							):




         '''simple docstring'''







         lowerCamelCase__:						Optional[int]							   = AutoTokenizer.from_pretrained(
             """hf-internal-testing/test_dynamic_tokenizer_legacy"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE							)
         self.assertTrue(tokenizer.special_attribute_present							)
         if is_tokenizers_available():
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizerFast"""							)

               # Test we can also load the slow version
               lowerCamelCase__:						Tuple							   = AutoTokenizer.from_pretrained(
                   """hf-internal-testing/test_dynamic_tokenizer_legacy"""    ,    trust_remote_code=_SCREAMING_SNAKE_CASE    ,    use_fast=_SCREAMING_SNAKE_CASE							)
               self.assertTrue(tokenizer.special_attribute_present							)
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizer"""							)
         else:
               self.assertEqual(tokenizer.__class__.__name__    ,    """NewTokenizer"""							)




   def 							lowerCamelCase_   (   self						:						Dict							):




         '''simple docstring'''







         with self.assertRaisesRegex(
             _SCREAMING_SNAKE_CASE    ,    """bert-base is not a local folder and is not a valid model identifier"""							):
               lowerCamelCase__:						Tuple							   = AutoTokenizer.from_pretrained("""bert-base"""							)




   def 							lowerCamelCase_   (   self						:						str							):




         '''simple docstring'''







         with self.assertRaisesRegex(
             _SCREAMING_SNAKE_CASE    ,    R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""							):
               lowerCamelCase__:						str							   = AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    ,    revision="""aaaaaa"""							)




   def 							lowerCamelCase_   (   self						:						List[Any]							):




         '''simple docstring'''







         lowerCamelCase__:						Optional[int]							   = AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
         with RequestCounter() as counter:
               lowerCamelCase__:						List[str]							   = AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""							)
               self.assertEqual(counter.get_request_count    ,    0							)
               self.assertEqual(counter.head_request_count    ,    1							)
               self.assertEqual(counter.other_request_count    ,    0							)












import numpy as np

import datasets


__a							     =					'\nCompute the Mahalanobis Distance\n\nMahalonobis distance is the distance between a point and a distribution.\nAnd not between two distinct points. It is effectively a multivariate equivalent of the Euclidean distance.\nIt was introduced by Prof. P. C. Mahalanobis in 1936\nand has been used in various statistical applications ever since\n[source: https://www.machinelearningplus.com/statistics/mahalanobis-distance/]\n'

__a							     =					'\\n@article{de2000mahalanobis,\n  title={The mahalanobis distance},\n  author={De Maesschalck, Roy and Jouan-Rimbaud, Delphine and Massart, D{\'e}sir{\'e} L},\n  journal={Chemometrics and intelligent laboratory systems},\n  volume={50},\n  number={1},\n  pages={1--18},\n  year={2000},\n  publisher={Elsevier}\n}\n'

__a							     =					'\nArgs:\n    X: List of datapoints to be compared with the `reference_distribution`.\n    reference_distribution: List of datapoints from the reference distribution we want to compare to.\nReturns:\n    mahalanobis: The Mahalonobis distance for each datapoint in `X`.\nExamples:\n\n    >>> mahalanobis_metric = datasets.load_metric("mahalanobis")\n    >>> results = mahalanobis_metric.compute(reference_distribution=[[0, 1], [1, 0]], X=[[0, 1]])\n    >>> print(results)\n    {\'mahalanobis\': array([0.5])}\n'

@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION							,		_KWARGS_DESCRIPTION  )
class     __a(	datasets.Metric  ):






    """simple docstring"""





    def 				a__ ( self    )					->						Union[str, Any]:
         return datasets.MetricInfo(
             description=_DESCRIPTION  ,citation=_CITATION  ,inputs_description=_KWARGS_DESCRIPTION  ,features=datasets.Features(
                 {
                     '''X''': datasets.Sequence(datasets.Value('''float'''  ,id='''sequence'''    )  ,id='''X'''    ),
                 }    )  ,)





    def 				a__ ( self  ,_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )					->						Any:
         # convert to numpy arrays
         UpperCAmelCase_     :				str  =     np.array(_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				List[Any]  =     np.array(_SCREAMING_SNAKE_CASE    )

         # Assert that arrays are 2D
         if len(X.shape    ) != 2:
              raise ValueError('''Expected `X` to be a 2D vector'''    )
         if len(reference_distribution.shape    ) != 2:
              raise ValueError('''Expected `reference_distribution` to be a 2D vector'''    )
         if reference_distribution.shape[0] < 2:
              raise ValueError(
                  '''Expected `reference_distribution` to be a 2D vector with more than one element in the first dimension'''    )

         # Get mahalanobis distance for each prediction
         UpperCAmelCase_     :				List[str]  =     X - np.mean(_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Dict  =     np.cov(reference_distribution.T    )
         try:
              UpperCAmelCase_     :				Any  =     np.linalg.inv(_SCREAMING_SNAKE_CASE    )
         except np.linalg.LinAlgError:
              UpperCAmelCase_     :				List[str]  =     np.linalg.pinv(_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Union[str, Any]  =     np.dot(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				List[Any]  =     np.dot(_SCREAMING_SNAKE_CASE  ,X_minus_mu.T    ).diagonal()

         return {"mahalanobis": mahal_dist}