

"""simple docstring"""




import inspect
import unittest
import warnings

from transformers import DeiTConfig
from transformers.models.auto import get_values
from transformers.testing_utils import (
    require_accelerate,
    require_torch,
    require_torch_gpu,
    require_vision,
    slow,
    torch_device,
)
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
						import torch
						from torch import nn

						from transformers import (
						    MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING,
						    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
						    MODEL_MAPPING,
						    DeiTForImageClassification,
						    DeiTForImageClassificationWithTeacher,
						    DeiTForMaskedImageModeling,
						    DeiTModel,
						)
						from transformers.models.deit.modeling_deit import DEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
						from PIL import Image

						from transformers import DeiTImageProcessor





class  lowerCamelCase    :

			def __init__(       self					:    Tuple     ,					__UpperCAmelCase					:    Union[str, Any]     ,					__UpperCAmelCase					:    Any=1_3     ,					__UpperCAmelCase					:    Optional[Any]=3_0     ,					__UpperCAmelCase					:    List[Any]=2     ,					__UpperCAmelCase					:    Tuple=3     ,					__UpperCAmelCase					:    List[str]=True     ,					__UpperCAmelCase					:    List[Any]=True     ,					__UpperCAmelCase					:    Optional[int]=3_2     ,					__UpperCAmelCase					:    Any=5     ,					__UpperCAmelCase					:    str=4     ,					__UpperCAmelCase					:    Any=3_7     ,					__UpperCAmelCase					:    Dict="gelu"     ,					__UpperCAmelCase					:    Union[str, Any]=0.1     ,					__UpperCAmelCase					:    Optional[int]=0.1     ,					__UpperCAmelCase					:    Dict=1_0     ,					__UpperCAmelCase					:    int=0.02     ,					__UpperCAmelCase					:    Optional[Any]=3     ,					__UpperCAmelCase					:    List[str]=None     ,					__UpperCAmelCase					:    List[Any]=2     ,					) ->     Any:
						SCREAMING_SNAKE_CASE__										=   parent
						SCREAMING_SNAKE_CASE__										=   batch_size
						SCREAMING_SNAKE_CASE__										=   image_size
						SCREAMING_SNAKE_CASE__										=   patch_size
						SCREAMING_SNAKE_CASE__										=   num_channels
						SCREAMING_SNAKE_CASE__										=   is_training
						SCREAMING_SNAKE_CASE__										=   use_labels
						SCREAMING_SNAKE_CASE__										=   hidden_size
						SCREAMING_SNAKE_CASE__										=   num_hidden_layers
						SCREAMING_SNAKE_CASE__										=   num_attention_heads
						SCREAMING_SNAKE_CASE__										=   intermediate_size
						SCREAMING_SNAKE_CASE__										=   hidden_act
						SCREAMING_SNAKE_CASE__										=   hidden_dropout_prob
						SCREAMING_SNAKE_CASE__										=   attention_probs_dropout_prob
						SCREAMING_SNAKE_CASE__										=   type_sequence_label_size
						SCREAMING_SNAKE_CASE__										=   initializer_range
						SCREAMING_SNAKE_CASE__										=   scope
						SCREAMING_SNAKE_CASE__										=   encoder_stride

						# in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
						SCREAMING_SNAKE_CASE__										=   (image_size // patch_size) ** 2
						SCREAMING_SNAKE_CASE__										=   num_patches + 2

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

						SCREAMING_SNAKE_CASE__										=   None
						if self.use_labels:
									SCREAMING_SNAKE_CASE__										=   ids_tensor([self.batch_size]     ,					self.type_sequence_label_size							)

						SCREAMING_SNAKE_CASE__										=   self.get_config()

						return config, pixel_values, labels

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Optional[int]:
						return DeiTConfig(
						    image_size=self.image_size     ,					patch_size=self.patch_size     ,					num_channels=self.num_channels     ,					hidden_size=self.hidden_size     ,					num_hidden_layers=self.num_hidden_layers     ,					num_attention_heads=self.num_attention_heads     ,					intermediate_size=self.intermediate_size     ,					hidden_act=self.hidden_act     ,					hidden_dropout_prob=self.hidden_dropout_prob     ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob     ,					is_decoder=__UpperCAmelCase     ,					initializer_range=self.initializer_range     ,					encoder_stride=self.encoder_stride     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    int     ,					__UpperCAmelCase					:    Optional[Any]     ,					__UpperCAmelCase					:    Union[str, Any]     ,					__UpperCAmelCase					:    Any							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   DeiTModel(config=__UpperCAmelCase							)
						model.to(__UpperCAmelCase							)
						model.eval()
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase							)
						self.parent.assertEqual(result.last_hidden_state.shape     ,					(self.batch_size, self.seq_length, self.hidden_size)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]     ,					__UpperCAmelCase					:    Any     ,					__UpperCAmelCase					:    Optional[Any]     ,					__UpperCAmelCase					:    str							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   DeiTForMaskedImageModeling(config=__UpperCAmelCase							)
						model.to(__UpperCAmelCase							)
						model.eval()
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase							)
						self.parent.assertEqual(
						    result.reconstruction.shape     ,					(self.batch_size, self.num_channels, self.image_size, self.image_size)							)

						# test greyscale images
						SCREAMING_SNAKE_CASE__										=   1
						SCREAMING_SNAKE_CASE__										=   DeiTForMaskedImageModeling(__UpperCAmelCase							)
						model.to(__UpperCAmelCase							)
						model.eval()

						SCREAMING_SNAKE_CASE__										=   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase							)
						self.parent.assertEqual(result.reconstruction.shape     ,					(self.batch_size, 1, self.image_size, self.image_size)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict     ,					__UpperCAmelCase					:    List[Any]     ,					__UpperCAmelCase					:    Any     ,					__UpperCAmelCase					:    Optional[int]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   self.type_sequence_label_size
						SCREAMING_SNAKE_CASE__										=   DeiTForImageClassification(__UpperCAmelCase							)
						model.to(__UpperCAmelCase							)
						model.eval()
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase     ,					labels=__UpperCAmelCase							)
						self.parent.assertEqual(result.logits.shape     ,					(self.batch_size, self.type_sequence_label_size)							)

						# test greyscale images
						SCREAMING_SNAKE_CASE__										=   1
						SCREAMING_SNAKE_CASE__										=   DeiTForImageClassification(__UpperCAmelCase							)
						model.to(__UpperCAmelCase							)
						model.eval()

						SCREAMING_SNAKE_CASE__										=   floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
						SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase     ,					labels=__UpperCAmelCase							)
						self.parent.assertEqual(result.logits.shape     ,					(self.batch_size, self.type_sequence_label_size)							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.prepare_config_and_inputs()
						(
						    (
						    SCREAMING_SNAKE_CASE__
						)				,		(
						    SCREAMING_SNAKE_CASE__
						)				,		(
						    SCREAMING_SNAKE_CASE__
						)				,		
						)										=   config_and_inputs
						SCREAMING_SNAKE_CASE__										=   {"""pixel_values""": pixel_values}
						return config, inputs_dict





@require_torch
class  lowerCamelCase    (A__      ,A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Any       		=				(
			    (
			        DeiTModel,
			        DeiTForImageClassification,
			        DeiTForImageClassificationWithTeacher,
			        DeiTForMaskedImageModeling,
			    )
			    if is_torch_available()
			    else ()
			)
			lowerCamelCase__ :		Tuple       		=				(
			    {
			        'feature-extraction': DeiTModel,
			        'image-classification': (DeiTForImageClassification, DeiTForImageClassificationWithTeacher),
			    }
			    if is_torch_available()
			    else {}
			)

			lowerCamelCase__ :		List[str]       		=				False
			lowerCamelCase__ :		Optional[Any]       		=				False
			lowerCamelCase__ :		Optional[int]       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   DeiTModelTester(self							)
						SCREAMING_SNAKE_CASE__										=   ConfigTester(self     ,					config_class=__UpperCAmelCase     ,					has_text_modality=__UpperCAmelCase     ,					hidden_size=3_7							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     Optional[int]:
						self.config_tester.run_common_tests()

			@unittest.skip(reason="""DeiT does not use inputs_embeds"""							)
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Tuple:
						pass

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     str:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
									SCREAMING_SNAKE_CASE__										=   model_class(__UpperCAmelCase							)
									self.assertIsInstance(model.get_input_embeddings()     ,					(nn.Module)							)
									SCREAMING_SNAKE_CASE__										=   model.get_output_embeddings()
									self.assertTrue(x is None or isinstance(__UpperCAmelCase     ,					nn.Linear							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     int:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs_for_common()

						for model_class in self.all_model_classes:
									SCREAMING_SNAKE_CASE__										=   model_class(__UpperCAmelCase							)
									SCREAMING_SNAKE_CASE__										=   inspect.signature(model.forward							)
									# signature.parameters is an OrderedDict => so arg_names order is deterministic
									SCREAMING_SNAKE_CASE__										=   [*signature.parameters.keys()]

									SCREAMING_SNAKE_CASE__										=   ["""pixel_values"""]
									self.assertListEqual(arg_names[:1]     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_model(*__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_masked_image_modeling(*__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs()
						self.model_tester.create_and_check_for_image_classification(*__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]     ,					__UpperCAmelCase					:    Dict     ,					__UpperCAmelCase					:    Optional[int]     ,					__UpperCAmelCase					:    List[Any]=False							) ->     str:
						SCREAMING_SNAKE_CASE__										=   super()._prepare_for_class(__UpperCAmelCase     ,					__UpperCAmelCase     ,					return_labels=__UpperCAmelCase							)

						if return_labels:
									if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
												del inputs_dict["labels"]

						return inputs_dict

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Union[str, Any]:
						if not self.model_tester.is_training:
									return

						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs_for_common()
						SCREAMING_SNAKE_CASE__										=   True

						for model_class in self.all_model_classes:
									# DeiTForImageClassificationWithTeacher supports inference-only
									if (
									    model_class in get_values(__UpperCAmelCase							)
									    or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
									):
												continue
									SCREAMING_SNAKE_CASE__										=   model_class(__UpperCAmelCase							)
									model.to(__UpperCAmelCase							)
									model.train()
									SCREAMING_SNAKE_CASE__										=   self._prepare_for_class(__UpperCAmelCase     ,					__UpperCAmelCase     ,					return_labels=__UpperCAmelCase							)
									SCREAMING_SNAKE_CASE__										=   model(**__UpperCAmelCase							).loss
									loss.backward()

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Dict:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs_for_common()
						if not self.model_tester.is_training:
									return

						SCREAMING_SNAKE_CASE__										=   False
						SCREAMING_SNAKE_CASE__										=   True

						for model_class in self.all_model_classes:
									if model_class in get_values(__UpperCAmelCase							) or not model_class.supports_gradient_checkpointing:
												continue
									# DeiTForImageClassificationWithTeacher supports inference-only
									if model_class.__name__ == "DeiTForImageClassificationWithTeacher":
												continue
									SCREAMING_SNAKE_CASE__										=   model_class(__UpperCAmelCase							)
									model.gradient_checkpointing_enable()
									model.to(__UpperCAmelCase							)
									model.train()
									SCREAMING_SNAKE_CASE__										=   self._prepare_for_class(__UpperCAmelCase     ,					__UpperCAmelCase     ,					return_labels=__UpperCAmelCase							)
									SCREAMING_SNAKE_CASE__										=   model(**__UpperCAmelCase							).loss
									loss.backward()

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Any:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.model_tester.prepare_config_and_inputs_for_common()

						SCREAMING_SNAKE_CASE__										=   [
						    {"""title""": """multi_label_classification""", """num_labels""": 2, """dtype""": torch.float},
						    {"""title""": """single_label_classification""", """num_labels""": 1, """dtype""": torch.long},
						    {"""title""": """regression""", """num_labels""": 1, """dtype""": torch.float},
						]

						for model_class in self.all_model_classes:
									if (
									    model_class
									    not in [
									        *get_values(__UpperCAmelCase							),
									        *get_values(__UpperCAmelCase							),
									    ]
									    or model_class.__name__ == "DeiTForImageClassificationWithTeacher"
									):
												continue

									for problem_type in problem_types:
												with self.subTest(msg=F"""Testing {model_class} with {problem_type["title"]}"""							):
															SCREAMING_SNAKE_CASE__										=   problem_type["""title"""]
															SCREAMING_SNAKE_CASE__										=   problem_type["""num_labels"""]

															SCREAMING_SNAKE_CASE__										=   model_class(__UpperCAmelCase							)
															model.to(__UpperCAmelCase							)
															model.train()

															SCREAMING_SNAKE_CASE__										=   self._prepare_for_class(__UpperCAmelCase     ,					__UpperCAmelCase     ,					return_labels=__UpperCAmelCase							)

															if problem_type["num_labels"] > 1:
																		SCREAMING_SNAKE_CASE__										=   inputs["""labels"""].unsqueeze(1							).repeat(1     ,					problem_type["""num_labels"""]							)

															SCREAMING_SNAKE_CASE__										=   inputs["""labels"""].to(problem_type["""dtype"""]							)

															# This tests that we do not trigger the warning form PyTorch "Using a target size that is different
															# to the input size. This will likely lead to incorrect results due to broadcasting. Please ensure
															# they have the same size." which is a symptom something in wrong for the regression problem.
															# See https://github.com/huggingface/transformers/issues/11780
															with warnings.catch_warnings(record=__UpperCAmelCase							) as warning_list:
																		SCREAMING_SNAKE_CASE__										=   model(**__UpperCAmelCase							).loss
															for w in warning_list:
																		if "Using a target size that is different to the input size" in str(w.message							):
																					raise ValueError(
																					    F"""Something is going wrong in the regression problem: intercepted {w.message}"""							)

															loss.backward()





			@slow
			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Union[str, Any]:
						for model_name in DEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
									SCREAMING_SNAKE_CASE__										=   DeiTModel.from_pretrained(__UpperCAmelCase							)
									self.assertIsNotNone(__UpperCAmelCase							)

def 						A							(  ):




			'''simple docstring'''


			SCREAMING_SNAKE_CASE__										=   Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""     )
			return image





@require_torch
@require_vision
class  lowerCamelCase    (unittest.TestCase      ):

			@cached_property
			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     List[str]:
						return (
						    DeiTImageProcessor.from_pretrained("""facebook/deit-base-distilled-patch16-224"""							)
						    if is_vision_available()
						    else None
						)

			@slow
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   DeiTForImageClassificationWithTeacher.from_pretrained("""facebook/deit-base-distilled-patch16-224"""							).to(
						    __UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   self.default_image_processor
						SCREAMING_SNAKE_CASE__										=   prepare_img()
						SCREAMING_SNAKE_CASE__										=   image_processor(images=__UpperCAmelCase     ,					return_tensors="""pt"""							).to(__UpperCAmelCase							)

						# forward pass
						with torch.no_grad():
									SCREAMING_SNAKE_CASE__										=   model(**__UpperCAmelCase							)

						# verify the logits
						SCREAMING_SNAKE_CASE__										=   torch.Size((1, 1_0_0_0)							)
						self.assertEqual(outputs.logits.shape     ,					__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   torch.tensor([-1.0_266, 0.1_912, -1.2_861]							).to(__UpperCAmelCase							)

						self.assertTrue(torch.allclose(outputs.logits[0, :3]     ,					__UpperCAmelCase     ,					atol=1e-4							)							)





			@slow
			@require_accelerate
			@require_torch_gpu
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   DeiTModel.from_pretrained(
						    """facebook/deit-base-distilled-patch16-224"""     ,					torch_dtype=torch.floataa     ,					device_map="""auto"""							)
						SCREAMING_SNAKE_CASE__										=   self.default_image_processor

						SCREAMING_SNAKE_CASE__										=   prepare_img()
						SCREAMING_SNAKE_CASE__										=   image_processor(images=__UpperCAmelCase     ,					return_tensors="""pt"""							)
						SCREAMING_SNAKE_CASE__										=   inputs.pixel_values.to(__UpperCAmelCase							)

						# forward pass to make sure inference works in fp16
						with torch.no_grad():
									SCREAMING_SNAKE_CASE__										=   model(__UpperCAmelCase							)





"""simple docstring"""




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Union[str, Any]       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		str       		=				False
			lowerCamelCase__ :		Optional[int]       		=				True

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     str:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   [
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """こんにちは""",
						    """こん""",
						    """にちは""",
						    """ばんは""",
						    """##こん""",
						    """##にちは""",
						    """##ばんは""",
						    """世界""",
						    """##世界""",
						    """、""",
						    """##、""",
						    """。""",
						    """##。""",
						]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]     ,					__UpperCAmelCase					:    Optional[int]							) ->     Tuple:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こんにちは 、 世界 。 こんばんは 、 世界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]     ,					__UpperCAmelCase					:    Union[str, Any]							) ->     int:
						SCREAMING_SNAKE_CASE__				,		SCREAMING_SNAKE_CASE__										=   self.get_input_output_texts(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode(__UpperCAmelCase     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.decode(__UpperCAmelCase     ,					clean_up_tokenization_spaces=__UpperCAmelCase							)
						return text, ids

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Tuple:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""mecab"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Union[str, Any]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic_lite"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(mecab_dic="""unidic"""							)
						except ModuleNotFoundError:
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(do_lower_case=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     List[str]:
						try:
									SCREAMING_SNAKE_CASE__										=   MecabTokenizer(
									    do_lower_case=__UpperCAmelCase     ,					normalize_text=__UpperCAmelCase     ,					mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""							)
						except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   MecabTokenizer(normalize_text=__UpperCAmelCase     ,					mecab_dic="""ipadic"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""sudachi"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    List[str]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""A"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国""", """人""", """参政""", """権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""B"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人""", """参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Optional[int]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(sudachi_dict_type="""core"""     ,					sudachi_split_mode="""C"""							)

						self.assertListEqual(tokenizer.tokenize("""外国人参政権"""							)     ,					["""外国人参政権"""]							)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(do_lower_case=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    Tuple							) ->     str:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(normalize_text=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]     ,					)

			@require_sudachi
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   SudachiTokenizer(trim_whitespace=__UpperCAmelCase     ,					sudachi_dict_type="""core"""							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[Any]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					word_tokenizer_type="""jumanpp"""							)
						self.assertIsNotNone(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。\nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize(__UpperCAmelCase							)
						self.assertListEqual(__UpperCAmelCase     ,					["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]							)
						self.assertListEqual(tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					"""tokenizer.bin"""							)
						with open(__UpperCAmelCase     ,					"""wb"""							) as handle:
									pickle.dump(__UpperCAmelCase     ,					__UpperCAmelCase							)

						with open(__UpperCAmelCase     ,					"""rb"""							) as handle:
									SCREAMING_SNAKE_CASE__										=   pickle.load(__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer_new.tokenize(__UpperCAmelCase							)

						self.assertListEqual(__UpperCAmelCase     ,					__UpperCAmelCase							)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     int:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(do_lower_case=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Union[str, Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(normalize_text=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer(trim_whitespace=__UpperCAmelCase							)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """							)     ,					["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]     ,					)

			@require_jumanpp
			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     Optional[Any]:
						SCREAMING_SNAKE_CASE__										=   JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""							)     ,					["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]     ,					)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     List[str]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   WordpieceTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こんにちは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは"""							)     ,					["""こん""", """##ばんは"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""							)     ,					["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    List[Any]							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.subword_tokenizer

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]							)

						SCREAMING_SNAKE_CASE__										=   subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""							)
						self.assertListEqual(__UpperCAmelCase     ,					["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (A__      ,unittest.TestCase      ):
			lowerCamelCase__ :		Dict       		=				BertJapaneseTokenizer
			lowerCamelCase__ :		Tuple       		=				False

			def        SCREAMING_SNAKE_CASE					(       self					:    int							) ->     List[Any]:
						super().setUp()

						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   os.path.join(self.tmpdirname     ,					VOCAB_FILES_NAMES["""vocab_file"""]							)
						with open(self.vocab_file     ,					"""w"""     ,					encoding="""utf-8"""							) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]							)							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]     ,					**__UpperCAmelCase					:    List[Any]							) ->     List[Any]:
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname     ,					subword_tokenizer_type="""character"""     ,					**__UpperCAmelCase							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Any     ,					__UpperCAmelCase					:    Any							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   """こんにちは、世界。 \nこんばんは、世界。"""
						SCREAMING_SNAKE_CASE__										=   """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
						return input_text, output_text

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Dict:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    str							) ->     Optional[Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Union[str, Any]							) ->     Union[str, Any]:
						pass  # TODO add if relevant

			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class(self.vocab_file     ,					subword_tokenizer_type="""character"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""							)
						self.assertListEqual(
						    __UpperCAmelCase     ,					["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]							)
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(__UpperCAmelCase							)     ,					[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     List[Any]:
						SCREAMING_SNAKE_CASE__										=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						SCREAMING_SNAKE_CASE__										=   {}
						for i, token in enumerate(__UpperCAmelCase							):
									SCREAMING_SNAKE_CASE__										=   i
						SCREAMING_SNAKE_CASE__										=   CharacterTokenizer(vocab=__UpperCAmelCase     ,					unk_token="""[UNK]"""							)

						self.assertListEqual(tokenizer.tokenize(""""""							)     ,					[]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちは"""							)     ,					["""こ""", """ん""", """に""", """ち""", """は"""]							)

						self.assertListEqual(tokenizer.tokenize("""こんにちほ"""							)     ,					["""こ""", """ん""", """に""", """ち""", """[UNK]"""]							)





			def        SCREAMING_SNAKE_CASE					(       self					:    Any							) ->     str:
						SCREAMING_SNAKE_CASE__										=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""ありがとう。"""     ,					add_special_tokens=__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.encode("""どういたしまして。"""     ,					add_special_tokens=__UpperCAmelCase							)

						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase							)
						SCREAMING_SNAKE_CASE__										=   tokenizer.build_inputs_with_special_tokens(__UpperCAmelCase     ,					__UpperCAmelCase							)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Optional[int]							) ->     Dict:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						SCREAMING_SNAKE_CASE__										=   AutoTokenizer.from_pretrained(__UpperCAmelCase							)
						self.assertIsInstance(__UpperCAmelCase     ,					__UpperCAmelCase							)





class  lowerCamelCase    (unittest.TestCase      ):

			def        SCREAMING_SNAKE_CASE					(       self					:    Dict							) ->     Any:
						SCREAMING_SNAKE_CASE__										=   """cl-tohoku/bert-base-japanese"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)
						SCREAMING_SNAKE_CASE__										=   """bert-base-cased"""
						with self.assertLogs("""transformers"""     ,					level="""WARNING"""							) as cm:
									BertJapaneseTokenizer.from_pretrained(__UpperCAmelCase							)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""							)							)


