

import unittest

from diffusers.models.unet_ad_blocks import *  # noqa F403
from diffusers.utils import torch_device

from .test_unet_blocks_common import UNetBlockTesterMixin

class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Optional[int]      		=       DownBlockaD  # noqa F405
      _lowerCamelCase       :			int      		=       'down'





      def 		__A	(   self							:				List[str]	):
            A_						  =				[-0.0_232, -0.9_869, 0.8_054, -0.0_637, -0.1_688, -1.4_264, 0.4_470, -1.3_394, 0.0_904]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Optional[int]      		=       ResnetDownsampleBlockaD  # noqa F405
      _lowerCamelCase       :			str      		=       'down'





      def 		__A	(   self							:				List[Any]	):
            A_						  =				[0.0_710, 0.2_410, -0.7_320, -1.0_757, -1.1_343, 0.3_540, -0.0_133, -0.2_576, 0.0_948]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Union[str, Any]      		=       AttnDownBlockaD  # noqa F405
      _lowerCamelCase       :			str      		=       'down'





      def 		__A	(   self							:				List[str]	):
            A_						  =				[0.0_636, 0.8_964, -0.6_234, -1.0_131, 0.0_844, 0.4_935, 0.3_437, 0.0_911, -0.2_957]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			str      		=       CrossAttnDownBlockaD  # noqa F405
      _lowerCamelCase       :			Optional[int]      		=       'down'





      def 		__A	(   self							:				Union[str, Any]	):
            A_  ,     A_						  =				super().prepare_init_args_and_inputs_for_common()
            A_						  =				32
            return init_dict, inputs_dict







      def 		__A	(   self							:				Union[str, Any]	):
            A_						  =				[0.2_238, -0.7_396, -0.2_255, -0.3_829, 0.1_925, 1.1_665, 0.0_603, -0.7_295, 0.1_983]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			str      		=       SimpleCrossAttnDownBlockaD  # noqa F405
      _lowerCamelCase       :			Dict      		=       'down'





      @property
      def 		__A	(   self							:				Optional[Any]	):
            return super().get_dummy_input(include_encoder_hidden_states=UpperCAmelCase	)





      def 		__A	(   self							:				str	):
            A_  ,     A_						  =				super().prepare_init_args_and_inputs_for_common()
            A_						  =				32
            return init_dict, inputs_dict







      @unittest.skipIf(torch_device == "mps"						,			"MPS result is not consistent"	)
      def 		__A	(   self							:				List[str]	):
            A_						  =				[0.7_921, -0.0_992, -0.1_962, -0.7_695, -0.4_242, 0.7_804, 0.4_737, 0.2_765, 0.3_338]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Optional[int]      		=       SkipDownBlockaD  # noqa F405
      _lowerCamelCase       :			Optional[Any]      		=       'down'





      @property
      def 		__A	(   self							:				List[Any]	):
            return super().get_dummy_input(include_skip_sample=UpperCAmelCase	)







      def 		__A	(   self							:				str	):
            A_						  =				[-0.0_845, -0.2_087, -0.2_465, 0.0_971, 0.1_900, -0.0_484, 0.2_664, 0.4_179, 0.5_069]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Optional[int]      		=       AttnSkipDownBlockaD  # noqa F405
      _lowerCamelCase       :			str      		=       'down'





      @property
      def 		__A	(   self							:				Any	):
            return super().get_dummy_input(include_skip_sample=UpperCAmelCase	)







      def 		__A	(   self							:				Union[str, Any]	):
            A_						  =				[0.5_539, 0.1_609, 0.4_924, 0.0_537, -0.1_995, 0.4_050, 0.0_979, -0.2_721, -0.0_642]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			List[str]      		=       DownEncoderBlockaD  # noqa F405
      _lowerCamelCase       :			Optional[Any]      		=       'down'





      @property
      def 		__A	(   self							:				Optional[int]	):
            return super().get_dummy_input(include_temb=UpperCAmelCase	)





      def 		__A	(   self							:				Optional[int]	):
            A_						  =				{
                "in_channels": 32,
                "out_channels": 32,
            }
            A_						  =				self.dummy_input
            return init_dict, inputs_dict







      def 		__A	(   self							:				Dict	):
            A_						  =				[1.1_102, 0.5_302, 0.4_872, -0.0_023, -0.8_042, 0.0_483, -0.3_489, -0.5_632, 0.7_626]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			str      		=       AttnDownEncoderBlockaD  # noqa F405
      _lowerCamelCase       :			Tuple      		=       'down'





      @property
      def 		__A	(   self							:				Dict	):
            return super().get_dummy_input(include_temb=UpperCAmelCase	)





      def 		__A	(   self							:				Optional[Any]	):
            A_						  =				{
                "in_channels": 32,
                "out_channels": 32,
            }
            A_						  =				self.dummy_input
            return init_dict, inputs_dict







      def 		__A	(   self							:				Optional[Any]	):
            A_						  =				[0.8_966, -0.1_486, 0.8_568, 0.8_141, -0.9_046, -0.1_342, -0.0_972, -0.7_417, 0.1_538]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Dict      		=       UNetMidBlockaD  # noqa F405
      _lowerCamelCase       :			int      		=       'mid'





      def 		__A	(   self							:				List[str]	):
            A_						  =				{
                "in_channels": 32,
                "temb_channels": 128,
            }
            A_						  =				self.dummy_input
            return init_dict, inputs_dict







      def 		__A	(   self							:				str	):
            A_						  =				[-0.1_062, 1.7_248, 0.3_494, 1.4_569, -0.0_910, -1.2_421, -0.9_984, 0.6_736, 1.0_028]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			str      		=       UNetMidBlockaDCrossAttn  # noqa F405
      _lowerCamelCase       :			Union[str, Any]      		=       'mid'





      def 		__A	(   self							:				Any	):
            A_  ,     A_						  =				super().prepare_init_args_and_inputs_for_common()
            A_						  =				32
            return init_dict, inputs_dict







      def 		__A	(   self							:				Optional[Any]	):
            A_						  =				[0.0_187, 2.4_220, 0.4_484, 1.1_203, -0.6_121, -1.5_122, -0.8_270, 0.7_851, 1.8_335]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Union[str, Any]      		=       UNetMidBlockaDSimpleCrossAttn  # noqa F405
      _lowerCamelCase       :			str      		=       'mid'





      @property
      def 		__A	(   self							:				Union[str, Any]	):
            return super().get_dummy_input(include_encoder_hidden_states=UpperCAmelCase	)





      def 		__A	(   self							:				Dict	):
            A_  ,     A_						  =				super().prepare_init_args_and_inputs_for_common()
            A_						  =				32
            return init_dict, inputs_dict







      def 		__A	(   self							:				Optional[int]	):
            A_						  =				[0.7_143, 1.9_974, 0.5_448, 1.3_977, 0.1_282, -1.1_237, -1.4_238, 0.5_530, 0.8_880]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Tuple      		=       UpBlockaD  # noqa F405
      _lowerCamelCase       :			Optional[Any]      		=       'up'





      @property
      def 		__A	(   self							:				int	):
            return super().get_dummy_input(include_res_hidden_states_tuple=UpperCAmelCase	)







      def 		__A	(   self							:				List[str]	):
            A_						  =				[-0.2_041, -0.4_165, -0.3_022, 0.0_041, -0.6_628, -0.7_053, 0.1_928, -0.0_325, 0.0_523]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			List[str]      		=       ResnetUpsampleBlockaD  # noqa F405
      _lowerCamelCase       :			int      		=       'up'





      @property
      def 		__A	(   self							:				int	):
            return super().get_dummy_input(include_res_hidden_states_tuple=UpperCAmelCase	)







      def 		__A	(   self							:				Tuple	):
            A_						  =				[0.2_287, 0.3_549, -0.1_346, 0.4_797, -0.1_715, -0.9_649, 0.7_305, -0.5_864, -0.6_244]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			str      		=       CrossAttnUpBlockaD  # noqa F405
      _lowerCamelCase       :			List[Any]      		=       'up'





      @property
      def 		__A	(   self							:				Union[str, Any]	):
            return super().get_dummy_input(include_res_hidden_states_tuple=UpperCAmelCase	)





      def 		__A	(   self							:				str	):
            A_  ,     A_						  =				super().prepare_init_args_and_inputs_for_common()
            A_						  =				32
            return init_dict, inputs_dict







      def 		__A	(   self							:				int	):
            A_						  =				[-0.1_403, -0.3_515, -0.0_420, -0.1_425, 0.3_167, 0.5_094, -0.2_181, 0.5_931, 0.5_582]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			List[str]      		=       SimpleCrossAttnUpBlockaD  # noqa F405
      _lowerCamelCase       :			int      		=       'up'





      @property
      def 		__A	(   self							:				List[Any]	):
            return super().get_dummy_input(include_res_hidden_states_tuple=UpperCAmelCase						,			include_encoder_hidden_states=UpperCAmelCase	)





      def 		__A	(   self							:				List[Any]	):
            A_  ,     A_						  =				super().prepare_init_args_and_inputs_for_common()
            A_						  =				32
            return init_dict, inputs_dict







      def 		__A	(   self							:				List[Any]	):
            A_						  =				[0.2_645, 0.1_480, 0.0_909, 0.8_044, -0.9_758, -0.9_083, 0.0_994, -1.1_453, -0.7_402]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Tuple      		=       AttnUpBlockaD  # noqa F405
      _lowerCamelCase       :			Optional[int]      		=       'up'





      @property
      def 		__A	(   self							:				Any	):
            return super().get_dummy_input(include_res_hidden_states_tuple=UpperCAmelCase	)







      @unittest.skipIf(torch_device == "mps"						,			"MPS result is not consistent"	)
      def 		__A	(   self							:				List[str]	):
            A_						  =				[0.0_979, 0.1_326, 0.0_021, 0.0_659, 0.2_249, 0.0_059, 0.1_132, 0.5_952, 0.1_033]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			int      		=       SkipUpBlockaD  # noqa F405
      _lowerCamelCase       :			Any      		=       'up'





      @property
      def 		__A	(   self							:				List[Any]	):
            return super().get_dummy_input(include_res_hidden_states_tuple=UpperCAmelCase	)







      def 		__A	(   self							:				str	):
            A_						  =				[-0.0_893, -0.1_234, -0.1_506, -0.0_332, 0.0_123, -0.0_211, 0.0_566, 0.0_143, 0.0_362]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Optional[int]      		=       AttnSkipUpBlockaD  # noqa F405
      _lowerCamelCase       :			str      		=       'up'





      @property
      def 		__A	(   self							:				str	):
            return super().get_dummy_input(include_res_hidden_states_tuple=UpperCAmelCase	)







      def 		__A	(   self							:				int	):
            A_						  =				[0.0_361, 0.0_617, 0.2_787, -0.0_350, 0.0_342, 0.3_421, -0.0_843, 0.0_913, 0.3_015]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Any      		=       UpDecoderBlockaD  # noqa F405
      _lowerCamelCase       :			List[str]      		=       'up'





      @property
      def 		__A	(   self							:				Optional[int]	):
            return super().get_dummy_input(include_temb=UpperCAmelCase	)





      def 		__A	(   self							:				int	):
            A_						  =				{"in_channels": 32, "out_channels": 32}

            A_						  =				self.dummy_input
            return init_dict, inputs_dict







      def 		__A	(   self							:				Any	):
            A_						  =				[0.4_404, 0.1_998, -0.9_886, -0.3_320, -0.3_128, -0.7_034, -0.6_955, -0.2_338, -0.3_137]
            super().test_output(UpperCAmelCase	)




class   _a					(			snake_case_     ,				unittest.TestCase				):




      """simple docstring"""

      _lowerCamelCase       :			Any      		=       AttnUpDecoderBlockaD  # noqa F405
      _lowerCamelCase       :			str      		=       'up'





      @property
      def 		__A	(   self							:				Optional[int]	):
            return super().get_dummy_input(include_temb=UpperCAmelCase	)





      def 		__A	(   self							:				Tuple	):
            A_						  =				{"in_channels": 32, "out_channels": 32}

            A_						  =				self.dummy_input
            return init_dict, inputs_dict







      def 		__A	(   self							:				Dict	):
            A_						  =				[0.6_738, 0.4_491, 0.1_055, 1.0_710, 0.7_316, 0.3_339, 0.3_352, 0.1_023, 0.3_568]
            super().test_output(UpperCAmelCase	)
"""simple docstring"""


import os
import shutil
import sys
import tempfile
import unittest
from pathlib import Path

import pytest

import transformers
from transformers import (
    BERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
    GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP,
    AutoTokenizer,
    BertConfig,
    BertTokenizer,
    BertTokenizerFast,
    CTRLTokenizer,
    GPTaTokenizer,
    GPTaTokenizerFast,
    PreTrainedTokenizerFast,
    RobertaTokenizer,
    RobertaTokenizerFast,
    is_tokenizers_available,
)
from transformers.models.auto.configuration_auto import CONFIG_MAPPING, AutoConfig
from transformers.models.auto.tokenization_auto import (
    TOKENIZER_MAPPING,
    get_tokenizer_config,
    tokenizer_class_from_name,
)
from transformers.models.roberta.configuration_roberta import RobertaConfig
from transformers.testing_utils import (
    DUMMY_DIFF_TOKENIZER_IDENTIFIER,
    DUMMY_UNKNOWN_IDENTIFIER,
    SMALL_MODEL_IDENTIFIER,
    RequestCounter,
    require_tokenizers,
    slow,
)


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
    from test_module.custom_tokenization_fast import CustomTokenizerFast


class 							a					(		unittest.TestCase ):


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      _UpperCAmelCase    			=   0


    @slow
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for model_name in (x for x in BERT_PRETRAINED_CONFIG_ARCHIVE_MAP.keys() if "japanese" not in x):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )

      for model_name in GPT2_PRETRAINED_CONFIG_ARCHIVE_MAP.keys():
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
        self.assertIsNotNone(__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					(GPTaTokenizer, GPTaTokenizerFast)       )
        self.assertGreater(len(__lowerCAmelCase       ) ,					0       )


    def   lowerCAmelCase_	(       self						:	Union[str, Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(RobertaTokenizer, RobertaTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					20       )


    def   lowerCAmelCase_	(       self						:	Any       ):
      _UpperCAmelCase    			=   AutoConfig.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )
      # Check that tokenizer_type ≠ model_type
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					config=__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      self.assertEqual(tokenizer.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2""" ,					use_fast=__lowerCAmelCase       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Tuple       ):
      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.txt""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""bert"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      with tempfile.TemporaryDirectory() as tmp_dir:
        shutil.copy("""./tests/fixtures/vocab.json""" ,					os.path.join(__lowerCAmelCase ,					"""vocab.json"""       )       )
        shutil.copy("""./tests/fixtures/merges.txt""" ,					os.path.join(__lowerCAmelCase ,					"""merges.txt"""       )       )

        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					tokenizer_type="""gpt2"""       )
        self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with pytest.raises(__lowerCAmelCase       ):
        AutoTokenizer.from_pretrained("""./""" ,					tokenizer_type="""xxx"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""wietsedv/bert-base-dutch-cased"""       )
        self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )

        if isinstance(__lowerCAmelCase ,					__lowerCAmelCase       ):
          self.assertEqual(tokenizer.basic_tokenizer.do_lower_case ,					__lowerCAmelCase       )
        else:
          self.assertEqual(tokenizer.do_lower_case ,					__lowerCAmelCase       )

        self.assertEqual(tokenizer.model_max_length ,					512       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):
      for tokenizer_class in [BertTokenizer, BertTokenizerFast, AutoTokenizer]:
        with self.assertRaisesRegex(
            __lowerCAmelCase ,					"""julien-c/herlolip-not-exists is not a local folder and is not a valid model identifier""" ,					):
          _UpperCAmelCase    			=   tokenizer_class.from_pretrained("""julien-c/herlolip-not-exists"""       )


    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      # tests: https://github.com/huggingface/transformers/pull/13251
      # 1. models with `-`, e.g. xlm-roberta -> xlm_roberta
      # 2. models that don't remap 1-1 from model-name to model file, e.g., openai-gpt -> openai
      _UpperCAmelCase    			=   TOKENIZER_MAPPING.values()
      _UpperCAmelCase    			=   []

      for slow_tok, fast_tok in tokenizers:
        if slow_tok is not None:
          tokenizer_names.append(slow_tok.__name__       )

        if fast_tok is not None:
          tokenizer_names.append(fast_tok.__name__       )

      for tokenizer_name in tokenizer_names:
        # must find the right class
        tokenizer_class_from_name(__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased""" ,					use_fast=__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertIsInstance(AutoTokenizer.from_pretrained("""bert-base-cased"""       ) ,					__lowerCAmelCase       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	int       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""distilbert-base-uncased""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   """Hello, world. How are you?"""
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""microsoft/mpnet-base""" ,					do_lower_case=__lowerCAmelCase       )
      _UpperCAmelCase    			=   tokenizer.tokenize(__lowerCAmelCase       )
      self.assertEqual("""[UNK]""" ,					tokens[0]       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	Optional[Any]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""robot-test/dummy-tokenizer-fast-with-model-config"""       )
      self.assertEqual(type(__lowerCAmelCase       ) ,					__lowerCAmelCase       )
      self.assertEqual(tokenizer.model_max_length ,					512       )
      self.assertEqual(tokenizer.vocab_size ,					3_0000       )
      self.assertEqual(tokenizer.unk_token ,					"""[UNK]"""       )
      self.assertEqual(tokenizer.padding_side ,					"""right"""       )
      self.assertEqual(tokenizer.truncation_side ,					"""right"""       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      self.assertIsInstance(__lowerCAmelCase ,					(BertTokenizer, BertTokenizerFast)       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )

      self.assertIsInstance(__lowerCAmelCase ,					tokenizer.__class__       )
      self.assertEqual(tokenizera.vocab_size ,					12       )


    def   lowerCAmelCase_	(       self						:	Optional[int]       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""ctrl"""       )
      # There is no fast CTRL so this always gives us a slow tokenizer.
      self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )


    def   lowerCAmelCase_	(       self						:	str       ):
      # Check we can load the tokenizer config of an online model.
      _UpperCAmelCase    			=   get_tokenizer_config("""bert-base-cased"""       )
      _UpperCAmelCase    			=   config.pop("""_commit_hash""" ,					__lowerCAmelCase       )
      # If we ever update bert-base-cased tokenizer config, this dict here will need to be updated.
      self.assertEqual(__lowerCAmelCase ,					{"""do_lower_case""": False}       )

      # This model does not have a tokenizer_config so we get back an empty dict.
      _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )
      self.assertDictEqual(__lowerCAmelCase ,					{}       )

      # A tokenizer saved with `save_pretrained` always creates a tokenizer config.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   get_tokenizer_config(__lowerCAmelCase       )

      # Check the class of the tokenizer was properly saved (note that it always saves the slow class).
      self.assertEqual(config["""tokenizer_class"""] ,					"""BertTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )

        _UpperCAmelCase    			=   CustomTokenizer.from_pretrained(__lowerCAmelCase       )
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )

        # Can register in two steps
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, None)       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        del TOKENIZER_MAPPING._extra_content[CustomConfig]
        # Can register in one step
        AutoTokenizer.register(
            __lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        self.assertEqual(TOKENIZER_MAPPING[CustomConfig] ,					(CustomTokenizer, CustomTokenizerFast)       )

        # Trying to register something existing in the Transformers library will raise an error
        with self.assertRaises(__lowerCAmelCase       ):
          AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )

        # We pass through a bert tokenizer fast cause there is no converter slow to fast for our new toknizer
        # and that model does not have a tokenizer.json
        with tempfile.TemporaryDirectory() as tmp_dir:
          _UpperCAmelCase    			=   BertTokenizerFast.from_pretrained(__lowerCAmelCase       )
          bert_tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   CustomTokenizerFast.from_pretrained(__lowerCAmelCase       )

        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
          self.assertIsInstance(__lowerCAmelCase ,					__lowerCAmelCase       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # If remote code is not set, we will time out when asking whether to load the model.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
      # If remote code is disabled, we can't load this config.
      with self.assertRaises(__lowerCAmelCase       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )

      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      # Test tokenizer can be reloaded.
      with tempfile.TemporaryDirectory() as tmp_dir:
        tokenizer.save_pretrained(__lowerCAmelCase       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(reloaded_tokenizer.special_attribute_present       )

      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        # Test tokenizer can be reloaded.
        with tempfile.TemporaryDirectory() as tmp_dir:
          tokenizer.save_pretrained(__lowerCAmelCase       )
          _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(reloaded_tokenizer.special_attribute_present       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertEqual(reloaded_tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    @require_tokenizers
    def   lowerCAmelCase_	(       self						:	List[str]       ):


      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Tuple  				=				False



      class 							a					(		lowerCAmelCase_ ):
          _snake_case  :	Optional[int]  				=				NewTokenizer
          _snake_case  :	Any  				=				False

      try:
        AutoConfig.register("""custom""" ,					__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					slow_tokenizer_class=__lowerCAmelCase       )
        AutoTokenizer.register(__lowerCAmelCase ,					fast_tokenizer_class=__lowerCAmelCase       )
        # If remote code is not set, the default is to use local
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer"""       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/test_dynamic_tokenizer""" ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote code is disabled, we load the local one.
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertFalse(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertFalse(tokenizer.special_attribute_present       )

        # If remote is enabled, we load from the Hub
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )
        self.assertTrue(tokenizer.special_attribute_present       )
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
        self.assertTrue(tokenizer.special_attribute_present       )

      finally:
        if "custom" in CONFIG_MAPPING._extra_content:
          del CONFIG_MAPPING._extra_content["custom"]
        if CustomConfig in TOKENIZER_MAPPING._extra_content:
          del TOKENIZER_MAPPING._extra_content[CustomConfig]


    def   lowerCAmelCase_	(       self						:	Dict       ):
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
          """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase       )
      self.assertTrue(tokenizer.special_attribute_present       )
      if is_tokenizers_available():
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizerFast"""       )

        # Test we can also load the slow version
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(
            """hf-internal-testing/test_dynamic_tokenizer_legacy""" ,					trust_remote_code=__lowerCAmelCase ,					use_fast=__lowerCAmelCase       )
        self.assertTrue(tokenizer.special_attribute_present       )
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )
      else:
        self.assertEqual(tokenizer.__class__.__name__ ,					"""NewTokenizer"""       )


    def   lowerCAmelCase_	(       self						:	List[Any]       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					"""bert-base is not a local folder and is not a valid model identifier"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""bert-base"""       )


    def   lowerCAmelCase_	(       self						:	int       ):
      with self.assertRaisesRegex(
          __lowerCAmelCase ,					R"""aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)"""       ):
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained(__lowerCAmelCase ,					revision="""aaaaaa"""       )


    def   lowerCAmelCase_	(       self						:	List[str]       ):
      # Make sure we have cached the tokenizer.
      _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
      with RequestCounter() as counter:
        _UpperCAmelCase    			=   AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-bert"""       )
        self.assertEqual(counter.get_request_count ,					0       )
        self.assertEqual(counter.head_request_count ,					1       )
        self.assertEqual(counter.other_request_count ,					0       )
