# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim

from dataclasses import dataclass
from typing import Optional, Tuple, Union

import flax
import jax
import jax.numpy as jnp

from ..configuration_utils import ConfigMixin, register_to_config
from .scheduling_utils_flax import (
    CommonSchedulerState,
    FlaxKarrasDiffusionSchedulers,
    FlaxSchedulerMixin,
    FlaxSchedulerOutput,
    add_noise_common,
    get_velocity_common,
)






@flax.struct.dataclass
class 		_lowerCamelCase		:
    __a						=   42

    # setable values
    __a						=   42
    __a						=   42
    __a						=   None
    @classmethod
    def     UpperCamelCase_					(	cls			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase )							->       Optional[Any]:
     return cls(common=lowerCAmelCase			,			init_noise_sigma=lowerCAmelCase			,			timesteps=lowerCAmelCase )






@dataclass
class 		_lowerCamelCase		( UpperCamelCase_     ):
    __a						=   42






class 		_lowerCamelCase		( UpperCamelCase_     ,  UpperCamelCase_     ):
    __a						=   [e.name for e in FlaxKarrasDiffusionSchedulers]

    __a						=   42
    @property
    def     UpperCamelCase_					(	self )							->       List[Any]:
     return True
    @register_to_config
    def __init__(	self			,			lowerCAmelCase = 1000			,			lowerCAmelCase = 0.0001			,			lowerCAmelCase = 0.02			,			lowerCAmelCase = "linear"			,			lowerCAmelCase = None			,			lowerCAmelCase = "fixed_small"			,			lowerCAmelCase = True			,			lowerCAmelCase = "epsilon"			,			lowerCAmelCase = jnp.floataa			,			)							->       Optional[int]:
     SCREAMING_SNAKE_CASE__: Optional[int]=   dtype
    def     UpperCamelCase_					(	self			,			lowerCAmelCase = None )							->       DDPMSchedulerState:
     if common is None:
      SCREAMING_SNAKE_CASE__: Optional[Any]=   CommonSchedulerState.create(self )

     # standard deviation of the initial noise distribution
     SCREAMING_SNAKE_CASE__: Dict=   jnp.array(1.0			,			dtype=self.dtype )

     SCREAMING_SNAKE_CASE__: int=   jnp.arange(0			,			self.config.num_train_timesteps ).round()[::-1]

     return DDPMSchedulerState.create(
         common=lowerCAmelCase			,			init_noise_sigma=lowerCAmelCase			,			timesteps=lowerCAmelCase			,			)
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase = None )							->       jnp.ndarray:
     return sample
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase = () )							->       DDPMSchedulerState:
     SCREAMING_SNAKE_CASE__: str=   self.config.num_train_timesteps // num_inference_steps
     # creates integer timesteps by multiplying by ratio
     # rounding to avoid issues when num_inference_step is power of 3
     SCREAMING_SNAKE_CASE__: str=   (jnp.arange(0			,			lowerCAmelCase ) * step_ratio).round()[::-1]

     return state.replace(
         num_inference_steps=lowerCAmelCase			,			timesteps=lowerCAmelCase			,			)
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase=None			,			lowerCAmelCase=None )							->       List[str]:
     SCREAMING_SNAKE_CASE__: Tuple=   state.common.alphas_cumprod[t]
     SCREAMING_SNAKE_CASE__: int=   jnp.where(t > 0			,			state.common.alphas_cumprod[t - 1]			,			jnp.array(1.0			,			dtype=self.dtype ) )

     # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf)
     # and sample from it to get previous sample
     # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample
     SCREAMING_SNAKE_CASE__: int=   (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * state.common.betas[t]

     if variance_type is None:
      SCREAMING_SNAKE_CASE__: Union[str, Any]=   self.config.variance_type

     # hacks - were probably added for training stability
     if variance_type == "fixed_small":
      SCREAMING_SNAKE_CASE__: Dict=   jnp.clip(lowerCAmelCase			,			a_min=1e-20 )
     # for rl-diffuser https://arxiv.org/abs/2205.09991
     elif variance_type == "fixed_small_log":
      SCREAMING_SNAKE_CASE__: str=   jnp.log(jnp.clip(lowerCAmelCase			,			a_min=1e-20 ) )
     elif variance_type == "fixed_large":
      SCREAMING_SNAKE_CASE__: Union[str, Any]=   state.common.betas[t]
     elif variance_type == "fixed_large_log":
      # Glide max_log
      SCREAMING_SNAKE_CASE__: Optional[Any]=   jnp.log(state.common.betas[t] )
     elif variance_type == "learned":
      return predicted_variance
     elif variance_type == "learned_range":
      SCREAMING_SNAKE_CASE__: List[Any]=   variance
      SCREAMING_SNAKE_CASE__: Any=   state.common.betas[t]
      SCREAMING_SNAKE_CASE__: List[Any]=   (predicted_variance + 1) / 2
      SCREAMING_SNAKE_CASE__: Optional[Any]=   frac * max_log + (1 - frac) * min_log

     return variance
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase = None			,			lowerCAmelCase = True			,			)							->       Union[FlaxDDPMSchedulerOutput, Tuple]:
     SCREAMING_SNAKE_CASE__: Union[str, Any]=   timestep

     if key is None:
      SCREAMING_SNAKE_CASE__: Optional[Any]=   jax.random.PRNGKey(0 )

     if model_output.shape[1] == sample.shape[1] * 2 and self.config.variance_type in ["learned", "learned_range"]:
      SCREAMING_SNAKE_CASE__,						SCREAMING_SNAKE_CASE__: Optional[int]=   jnp.split(lowerCAmelCase			,			sample.shape[1]			,			axis=1 )
     else:
      SCREAMING_SNAKE_CASE__: Any=   None

     # 1. compute alphas, betas
     SCREAMING_SNAKE_CASE__: List[Any]=   state.common.alphas_cumprod[t]
     SCREAMING_SNAKE_CASE__: Optional[int]=   jnp.where(t > 0			,			state.common.alphas_cumprod[t - 1]			,			jnp.array(1.0			,			dtype=self.dtype ) )
     SCREAMING_SNAKE_CASE__: Optional[int]=   1 - alpha_prod_t
     SCREAMING_SNAKE_CASE__: str=   1 - alpha_prod_t_prev

     # 2. compute predicted original sample from predicted noise also called
     # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf
     if self.config.prediction_type == "epsilon":
      SCREAMING_SNAKE_CASE__: Dict=   (sample - beta_prod_t ** 0.5 * model_output) / alpha_prod_t ** 0.5
     elif self.config.prediction_type == "sample":
      SCREAMING_SNAKE_CASE__: str=   model_output
     elif self.config.prediction_type == "v_prediction":
      SCREAMING_SNAKE_CASE__: Tuple=   (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output
     else:
      raise ValueError(
          f'prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` '
          ''' for the FlaxDDPMScheduler.''' )

     # 3. Clip "predicted x_0"
     if self.config.clip_sample:
      SCREAMING_SNAKE_CASE__: Any=   jnp.clip(lowerCAmelCase			,			-1			,			1 )

     # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t
     # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
     SCREAMING_SNAKE_CASE__: int=   (alpha_prod_t_prev ** 0.5 * state.common.betas[t]) / beta_prod_t
     SCREAMING_SNAKE_CASE__: Any=   state.common.alphas[t] ** 0.5 * beta_prod_t_prev / beta_prod_t

     # 5. Compute predicted previous sample µ_t
     # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf
     SCREAMING_SNAKE_CASE__: Dict=   pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample

     # 6. Add noise
     def random_variance():
      SCREAMING_SNAKE_CASE__: int=   jax.random.split(lowerCAmelCase			,			num=1 )
      SCREAMING_SNAKE_CASE__: str=   jax.random.normal(lowerCAmelCase			,			shape=model_output.shape			,			dtype=self.dtype )
      return (self._get_variance(lowerCAmelCase			,			lowerCAmelCase			,			predicted_variance=lowerCAmelCase ) ** 0.5) * noise

     SCREAMING_SNAKE_CASE__: Union[str, Any]=   jnp.where(t > 0			,			random_variance()			,			jnp.zeros(model_output.shape			,			dtype=self.dtype ) )

     SCREAMING_SNAKE_CASE__: Optional[int]=   pred_prev_sample + variance

     if not return_dict:
      return (pred_prev_sample, state)

     return FlaxDDPMSchedulerOutput(prev_sample=lowerCAmelCase			,			state=lowerCAmelCase )
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase			,			)							->       jnp.ndarray:
     return add_noise_common(state.common			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase )
    def     UpperCamelCase_					(	self			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase			,			)							->       jnp.ndarray:
     return get_velocity_common(state.common			,			lowerCAmelCase			,			lowerCAmelCase			,			lowerCAmelCase )



    def __len__(	self )							->       Tuple:
     return self.config.num_train_timesteps







import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / 'utils'))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast

class     __a(	unittest.TestCase  ):






    """simple docstring"""





    def 				a__ ( self    )					->						Union[str, Any]:
         UpperCAmelCase_     :				Tuple  =     0





    @slow
    def 				a__ ( self    )					->						Any:
         for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
              UpperCAmelCase_     :				Optional[Any]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
              self.assertIsNotNone(_SCREAMING_SNAKE_CASE    )
              self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,(BertTokenizer, BertTokenizerFast)    )
              self.assertGreater(len(_SCREAMING_SNAKE_CASE    )  ,0    )

         for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
              UpperCAmelCase_     :				Union[str, Any]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
              self.assertIsNotNone(_SCREAMING_SNAKE_CASE    )
              self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,(GPTaTokenizer, GPTaTokenizerFast)    )
              self.assertGreater(len(_SCREAMING_SNAKE_CASE    )  ,0    )





    def 				a__ ( self    )					->						Optional[Any]:
         UpperCAmelCase_     :				List[str]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
         self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,(BertTokenizer, BertTokenizerFast)    )
         self.assertEqual(tokenizer.vocab_size  ,12    )





    def 				a__ ( self    )					->						Tuple:
         UpperCAmelCase_     :				Any  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
         self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,(RobertaTokenizer, RobertaTokenizerFast)    )
         self.assertEqual(tokenizer.vocab_size  ,20    )





    def 				a__ ( self    )					->						List[str]:
         UpperCAmelCase_     :				int  =     AutoConfig.from_pretrained(_SCREAMING_SNAKE_CASE    )
         self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )
         # Check that tokenizer_type ≠ model_type
         UpperCAmelCase_     :				Optional[int]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,config=_SCREAMING_SNAKE_CASE    )
         self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,(BertTokenizer, BertTokenizerFast)    )
         self.assertEqual(tokenizer.vocab_size  ,12    )





    def 				a__ ( self    )					->						Dict:
         with tempfile.TemporaryDirectory() as tmp_dir:
              shutil.copy('''./tests/fixtures/vocab.txt'''  ,os.path.join(_SCREAMING_SNAKE_CASE  ,'''vocab.txt'''    )    )

              UpperCAmelCase_     :				Optional[int]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,tokenizer_type='''bert'''  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )

         with tempfile.TemporaryDirectory() as tmp_dir:
              shutil.copy('''./tests/fixtures/vocab.json'''  ,os.path.join(_SCREAMING_SNAKE_CASE  ,'''vocab.json'''    )    )
              shutil.copy('''./tests/fixtures/merges.txt'''  ,os.path.join(_SCREAMING_SNAKE_CASE  ,'''merges.txt'''    )    )

              UpperCAmelCase_     :				Optional[Any]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,tokenizer_type='''gpt2'''  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )





    @require_tokenizers
    def 				a__ ( self    )					->						Optional[int]:
         with tempfile.TemporaryDirectory() as tmp_dir:
              shutil.copy('''./tests/fixtures/vocab.txt'''  ,os.path.join(_SCREAMING_SNAKE_CASE  ,'''vocab.txt'''    )    )

              UpperCAmelCase_     :				Tuple  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,tokenizer_type='''bert'''    )
              self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )

         with tempfile.TemporaryDirectory() as tmp_dir:
              shutil.copy('''./tests/fixtures/vocab.json'''  ,os.path.join(_SCREAMING_SNAKE_CASE  ,'''vocab.json'''    )    )
              shutil.copy('''./tests/fixtures/merges.txt'''  ,os.path.join(_SCREAMING_SNAKE_CASE  ,'''merges.txt'''    )    )

              UpperCAmelCase_     :				Union[str, Any]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,tokenizer_type='''gpt2'''    )
              self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )





    def 				a__ ( self    )					->						int:
         with pytest.raises(_SCREAMING_SNAKE_CASE    ):
              AutoTokenizer.from_pretrained('''./'''  ,tokenizer_type='''xxx'''    )





    @require_tokenizers
    def 				a__ ( self    )					->						Optional[Any]:
         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
              UpperCAmelCase_     :				Any  =     tokenizer_class.from_pretrained('''wietsedv/bert-base-dutch-cased'''    )
              self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,(BertTokenizer, BertTokenizerFast)    )

              if isinstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    ):
                   self.assertEqual(tokenizer.basic_tokenizer.do_lower_case  ,_SCREAMING_SNAKE_CASE    )
              else:
                   self.assertEqual(tokenizer.do_lower_case  ,_SCREAMING_SNAKE_CASE    )

              self.assertEqual(tokenizer.model_max_length  ,512    )





    @require_tokenizers
    def 				a__ ( self    )					->						List[Any]:
         for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
              with self.assertRaisesRegex(
                  _SCREAMING_SNAKE_CASE  ,'''julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier'''  ,):
                   UpperCAmelCase_     :				int  =     tokenizer_class.from_pretrained('''julien-c/herlolip-not-exists'''    )





    def 				a__ ( self    )					->						Optional[Any]:
         # tests: https://github.com/huggingface/transformers/pull/13251
         # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
         # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
         UpperCAmelCase_     :				int  =     TOKENIZER_MAPPING.values()
         UpperCAmelCase_     :				List[Any]  =     []

         for slow_tok, fast_tok in tokenizers:
              if slow_tok is not None:
                   tokenizer_names.append(slow_tok.__name__    )

              if fast_tok is not None:
                   tokenizer_names.append(fast_tok.__name__    )

         for tokenizer_name in tokenizer_names:
              # must find the right class
              tokenizer_class_from_name(_SCREAMING_SNAKE_CASE    )





    @require_tokenizers
    def 				a__ ( self    )					->						Tuple:
         self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''  ,use_fast=_SCREAMING_SNAKE_CASE    )  ,_SCREAMING_SNAKE_CASE    )
         self.assertIsInstance(AutoTokenizer.from_pretrained('''bert-base-cased'''    )  ,_SCREAMING_SNAKE_CASE    )





    @require_tokenizers
    def 				a__ ( self    )					->						Optional[int]:
         UpperCAmelCase_     :				str  =     AutoTokenizer.from_pretrained('''distilbert-base-uncased'''  ,do_lower_case=_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				int  =     '''Hello, world. How are you?'''
         UpperCAmelCase_     :				List[Any]  =     tokenizer.tokenize(_SCREAMING_SNAKE_CASE    )
         self.assertEqual('''[UNK]'''  ,tokens[0]    )

         UpperCAmelCase_     :				Any  =     AutoTokenizer.from_pretrained('''microsoft/mpnet-base'''  ,do_lower_case=_SCREAMING_SNAKE_CASE    )
         UpperCAmelCase_     :				Tuple  =     tokenizer.tokenize(_SCREAMING_SNAKE_CASE    )
         self.assertEqual('''[UNK]'''  ,tokens[0]    )





    @require_tokenizers
    def 				a__ ( self    )					->						Dict:
         UpperCAmelCase_     :				List[Any]  =     AutoTokenizer.from_pretrained('''robot-test/dummy-tokenizer-fast-with-model-config'''    )
         self.assertEqual(type(_SCREAMING_SNAKE_CASE    )  ,_SCREAMING_SNAKE_CASE    )
         self.assertEqual(tokenizer.model_max_length  ,512    )
         self.assertEqual(tokenizer.vocab_size  ,30_000    )
         self.assertEqual(tokenizer.unk_token  ,'''[UNK]'''    )
         self.assertEqual(tokenizer.padding_side  ,'''right'''    )
         self.assertEqual(tokenizer.truncation_side  ,'''right'''    )





    def 				a__ ( self    )					->						Dict:
         UpperCAmelCase_     :				Any  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
         self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,(BertTokenizer, BertTokenizerFast)    )
         with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				Dict  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )

         self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,tokenizer.__class__    )
         self.assertEqual(tokenizera.vocab_size  ,12    )





    def 				a__ ( self    )					->						Optional[Any]:
         UpperCAmelCase_     :				Tuple  =     AutoTokenizer.from_pretrained('''ctrl'''    )
         # There is no fast CTRL so this always gives us a slow tokenizer.
         self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )





    def 				a__ ( self    )					->						str:
         # Check we can load the tokenizer config of an online model.
         UpperCAmelCase_     :				int  =     get_tokenizer_config('''bert-base-cased'''    )
         UpperCAmelCase_     :				Optional[int]  =     config.pop('''_commit_hash'''  ,_SCREAMING_SNAKE_CASE    )
         # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
         self.assertEqual(_SCREAMING_SNAKE_CASE  ,{'''do_lower_case''': False}    )

         # This model does not have a tokenizer_config so we get back an empty dict.
         UpperCAmelCase_     :				Any  =     get_tokenizer_config(_SCREAMING_SNAKE_CASE    )
         self.assertDictEqual(_SCREAMING_SNAKE_CASE  ,{}    )

         # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
         UpperCAmelCase_     :				Dict  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
         with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				Dict  =     get_tokenizer_config(_SCREAMING_SNAKE_CASE    )

         # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
         self.assertEqual(config['''tokenizer_class''']  ,'''BertTokenizer'''    )





    def 				a__ ( self    )					->						str:
         try:
              AutoConfig.register('''custom'''  ,_SCREAMING_SNAKE_CASE    )

              AutoTokenizer.register(_SCREAMING_SNAKE_CASE  ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE    )
              # Trying to register something existing in the Transformers library will raise an error
              with self.assertRaises(_SCREAMING_SNAKE_CASE    ):
                   AutoTokenizer.register(_SCREAMING_SNAKE_CASE  ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE    )

              UpperCAmelCase_     :				str  =     CustomTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
              with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )

                   UpperCAmelCase_     :				Dict  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
                   self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )

         finally:
              if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
              if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]





    @require_tokenizers
    def 				a__ ( self    )					->						int:
         try:
              AutoConfig.register('''custom'''  ,_SCREAMING_SNAKE_CASE    )

              # Can register in two steps
              AutoTokenizer.register(_SCREAMING_SNAKE_CASE  ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(TOKENIZER_MAPPING[CustomConfig]  ,(CustomTokenizer, None)    )
              AutoTokenizer.register(_SCREAMING_SNAKE_CASE  ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(TOKENIZER_MAPPING[CustomConfig]  ,(CustomTokenizer, CustomTokenizerFast)    )

              del TOKENIZER_MAPPING._extra_content[CustomConfig]
              # Can register in one step
              AutoTokenizer.register(
                  _SCREAMING_SNAKE_CASE  ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE  ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(TOKENIZER_MAPPING[CustomConfig]  ,(CustomTokenizer, CustomTokenizerFast)    )

              # Trying to register something existing in the Transformers library will raise an error
              with self.assertRaises(_SCREAMING_SNAKE_CASE    ):
                   AutoTokenizer.register(_SCREAMING_SNAKE_CASE  ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE    )

              # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
              # and that model does not have a tokenizer.json
              with tempfile.TemporaryDirectory() as tmp_dir:
                   UpperCAmelCase_     :				List[str]  =     BertTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE    )
                   bert_tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )
                   UpperCAmelCase_     :				str  =     CustomTokenizerFast.from_pretrained(_SCREAMING_SNAKE_CASE    )

              with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )

                   UpperCAmelCase_     :				Optional[int]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE    )
                   self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )

                   UpperCAmelCase_     :				Any  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,use_fast=_SCREAMING_SNAKE_CASE    )
                   self.assertIsInstance(_SCREAMING_SNAKE_CASE  ,_SCREAMING_SNAKE_CASE    )

         finally:
              if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
              if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]





    def 				a__ ( self    )					->						Optional[int]:
         # If remote code is not set, we will time out when asking whether to load the model.
         with self.assertRaises(_SCREAMING_SNAKE_CASE    ):
              UpperCAmelCase_     :				Tuple  =     AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''    )
         # If remote code is disabled, we can't load this config.
         with self.assertRaises(_SCREAMING_SNAKE_CASE    ):
              UpperCAmelCase_     :				Union[str, Any]  =     AutoTokenizer.from_pretrained(
                  '''hf-internal-testing/test_dynamic_tokenizer'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE    )

         UpperCAmelCase_     :				Dict  =     AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE    )
         self.assertTrue(tokenizer.special_attribute_present    )
         # Test tokenizer can be reloaded.
         with tempfile.TemporaryDirectory() as tmp_dir:
              tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )
              UpperCAmelCase_     :				int  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,trust_remote_code=_SCREAMING_SNAKE_CASE    )
         self.assertTrue(reloaded_tokenizer.special_attribute_present    )

         if is_tokenizers_available():
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizerFast'''    )
              self.assertEqual(reloaded_tokenizer.__class__.__name__  ,'''NewTokenizerFast'''    )

              # Test we can also load the slow version
              UpperCAmelCase_     :				List[str]  =     AutoTokenizer.from_pretrained(
                  '''hf-internal-testing/test_dynamic_tokenizer'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertTrue(tokenizer.special_attribute_present    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizer'''    )
              # Test tokenizer can be reloaded.
              with tempfile.TemporaryDirectory() as tmp_dir:
                   tokenizer.save_pretrained(_SCREAMING_SNAKE_CASE    )
                   UpperCAmelCase_     :				Optional[int]  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,trust_remote_code=_SCREAMING_SNAKE_CASE  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(reloaded_tokenizer.__class__.__name__  ,'''NewTokenizer'''    )
              self.assertTrue(reloaded_tokenizer.special_attribute_present    )
         else:
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizer'''    )
              self.assertEqual(reloaded_tokenizer.__class__.__name__  ,'''NewTokenizer'''    )





    @require_tokenizers
    def 				a__ ( self    )					->						Optional[int]:

         class     __a(	_a  ):






             """simple docstring"""





             lowerCAmelCase         = False


         class     __a(	_a  ):






             """simple docstring"""





             lowerCAmelCase         = NewTokenizer
             lowerCAmelCase         = False

         try:
              AutoConfig.register('''custom'''  ,_SCREAMING_SNAKE_CASE    )
              AutoTokenizer.register(_SCREAMING_SNAKE_CASE  ,slow_tokenizer_class=_SCREAMING_SNAKE_CASE    )
              AutoTokenizer.register(_SCREAMING_SNAKE_CASE  ,fast_tokenizer_class=_SCREAMING_SNAKE_CASE    )
              # If remote code is not set, the default is to use local
              UpperCAmelCase_     :				int  =     AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizerFast'''    )
              self.assertFalse(tokenizer.special_attribute_present    )
              UpperCAmelCase_     :				Any  =     AutoTokenizer.from_pretrained('''hf-internal-testing/test_dynamic_tokenizer'''  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizer'''    )
              self.assertFalse(tokenizer.special_attribute_present    )

              # If remote code is disabled, we load the local one.
              UpperCAmelCase_     :				Tuple  =     AutoTokenizer.from_pretrained(
                  '''hf-internal-testing/test_dynamic_tokenizer'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizerFast'''    )
              self.assertFalse(tokenizer.special_attribute_present    )
              UpperCAmelCase_     :				List[str]  =     AutoTokenizer.from_pretrained(
                  '''hf-internal-testing/test_dynamic_tokenizer'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizer'''    )
              self.assertFalse(tokenizer.special_attribute_present    )

              # If remote is enabled, we load from the Hub
              UpperCAmelCase_     :				Optional[int]  =     AutoTokenizer.from_pretrained(
                  '''hf-internal-testing/test_dynamic_tokenizer'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizerFast'''    )
              self.assertTrue(tokenizer.special_attribute_present    )
              UpperCAmelCase_     :				Any  =     AutoTokenizer.from_pretrained(
                  '''hf-internal-testing/test_dynamic_tokenizer'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizer'''    )
              self.assertTrue(tokenizer.special_attribute_present    )

         finally:
              if "custom" in CONFIG_MAPPING._extra_content:
                   del CONFIG_MAPPING._extra_content["custom"]
              if CustomConfig in TOKENIZER_MAPPING._extra_content:
                   del TOKENIZER_MAPPING._extra_content[CustomConfig]





    def 				a__ ( self    )					->						int:
         UpperCAmelCase_     :				Optional[int]  =     AutoTokenizer.from_pretrained(
             '''hf-internal-testing/test_dynamic_tokenizer_legacy'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE    )
         self.assertTrue(tokenizer.special_attribute_present    )
         if is_tokenizers_available():
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizerFast'''    )

              # Test we can also load the slow version
              UpperCAmelCase_     :				Tuple  =     AutoTokenizer.from_pretrained(
                  '''hf-internal-testing/test_dynamic_tokenizer_legacy'''  ,trust_remote_code=_SCREAMING_SNAKE_CASE  ,use_fast=_SCREAMING_SNAKE_CASE    )
              self.assertTrue(tokenizer.special_attribute_present    )
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizer'''    )
         else:
              self.assertEqual(tokenizer.__class__.__name__  ,'''NewTokenizer'''    )





    def 				a__ ( self    )					->						Optional[Any]:
         with self.assertRaisesRegex(
             _SCREAMING_SNAKE_CASE  ,'''bert-base is not a local folder and is not a valid model identifier'''    ):
              UpperCAmelCase_     :				Tuple  =     AutoTokenizer.from_pretrained('''bert-base'''    )





    def 				a__ ( self    )					->						List[Any]:
         with self.assertRaisesRegex(
             _SCREAMING_SNAKE_CASE  ,R'''aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'''    ):
              UpperCAmelCase_     :				str  =     AutoTokenizer.from_pretrained(_SCREAMING_SNAKE_CASE  ,revision='''aaaaaa'''    )





    def 				a__ ( self    )					->						Any:
         # Make sure we have cached the tokenizer.
         UpperCAmelCase_     :				Optional[int]  =     AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
         with RequestCounter() as counter:
              UpperCAmelCase_     :				List[str]  =     AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
              self.assertEqual(counter.get_request_count  ,0    )
              self.assertEqual(counter.head_request_count  ,1    )
              self.assertEqual(counter.other_request_count  ,0    )