





from __future__ import annotations

import unittest

import numpy as np

from transformers import OPTConfig, is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
       import tensorflow as tf

       from transformers import GPTaTokenizer, TFOPTForCausalLM, TFOPTModel






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase=None,       __lowerCamelCase=None	):
     if attention_mask is None:
          _SCREAMING_SNAKE_CASE       :	Any						    =							tf.cast(tf.math.not_equal(UpperCamelCase__,       config.pad_token_id	),       tf.inta	)
     return {"input_ids": input_ids, "attention_mask": attention_mask}





@require_tf
class 			lowerCAmelCase__:






       '''simple docstring'''


       __snake_case 			=    OPTConfig
       __snake_case 			=    {}
       __snake_case 			=    '''gelu'''




       def __init__(      self    ,						__lowerCamelCase    ,						__lowerCamelCase=1_3    ,						__lowerCamelCase=7    ,						__lowerCamelCase=True    ,						__lowerCamelCase=False    ,						__lowerCamelCase=9_9    ,						__lowerCamelCase=1_6    ,						__lowerCamelCase=2    ,						__lowerCamelCase=4    ,						__lowerCamelCase=4    ,						__lowerCamelCase="gelu"    ,						__lowerCamelCase=0.1    ,						__lowerCamelCase=0.1    ,						__lowerCamelCase=2_0    ,						__lowerCamelCase=2    ,						__lowerCamelCase=1    ,						__lowerCamelCase=0    ,						__lowerCamelCase=1_6    ,						__lowerCamelCase=1_6    ,						)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							parent
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							batch_size
            _SCREAMING_SNAKE_CASE       :	int						    =							seq_length
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							is_training
            _SCREAMING_SNAKE_CASE       :	int						    =							use_labels
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							vocab_size
            _SCREAMING_SNAKE_CASE       :	Any						    =							hidden_size
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							num_hidden_layers
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							num_attention_heads
            _SCREAMING_SNAKE_CASE       :	Dict						    =							intermediate_size
            _SCREAMING_SNAKE_CASE       :	str						    =							hidden_act
            _SCREAMING_SNAKE_CASE       :	int						    =							hidden_dropout_prob
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							attention_probs_dropout_prob
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							max_position_embeddings
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							eos_token_id
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							pad_token_id
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							bos_token_id
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							embed_dim
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							word_embed_proj_dim
            _SCREAMING_SNAKE_CASE       :	str						    =							False




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	int						    =							ids_tensor([self.batch_size, self.seq_length - 1]    ,						self.vocab_size							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size							)    ,						1							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tf.concat([input_ids, eos_tensor]    ,						axis=1							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							self.config_cls(
                vocab_size=self.vocab_size    ,						hidden_size=self.hidden_size    ,						num_hidden_layers=self.num_hidden_layers    ,						num_attention_heads=self.num_attention_heads    ,						ffn_dim=self.intermediate_size    ,						dropout=self.hidden_dropout_prob    ,						attention_dropout=self.attention_probs_dropout_prob    ,						max_position_embeddings=self.max_position_embeddings    ,						eos_token_id=self.eos_token_id    ,						bos_token_id=self.bos_token_id    ,						pad_token_id=self.pad_token_id    ,						embed_dim=self.embed_dim    ,						word_embed_proj_dim=self.word_embed_proj_dim    ,						is_encoder_decoder=__a    ,						**self.config_updates    ,						)
            _SCREAMING_SNAKE_CASE       :	int						    =							prepare_opt_inputs_dict(__a    ,						__a							)
            return config, inputs_dict







       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	str						    =							TFOPTModel(config=__a							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							inputs_dict["""input_ids"""]

            _SCREAMING_SNAKE_CASE       :	Any						    =							input_ids[:1, :]
            _SCREAMING_SNAKE_CASE       :	str						    =							inputs_dict["""attention_mask"""][:1, :]
            _SCREAMING_SNAKE_CASE       :	Dict						    =							1

            # first forward pass
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							model(__a    ,						attention_mask=__a    ,						use_cache=__a							)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							outputs.to_tuple()

            # create hypothetical next token and extent to next_input_ids
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							ids_tensor((self.batch_size, 3)    ,						config.vocab_size							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tf.cast(ids_tensor((self.batch_size, 3)    ,						2							)    ,						tf.inta							)

            # append to next input_ids and
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							tf.concat([input_ids, next_tokens]    ,						axis=-1							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							tf.concat([attention_mask, next_attn_mask]    ,						axis=-1							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							model(__a    ,						attention_mask=__a							)[0]
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							model(__a    ,						attention_mask=__a    ,						past_key_values=__a							)[0]

            self.parent.assertEqual(next_tokens.shape[1]    ,						output_from_past.shape[1]							)

            # select random slice
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							int(ids_tensor((1,)    ,						output_from_past.shape[-1]							)							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							output_from_no_past[:, -3:, random_slice_idx]
            _SCREAMING_SNAKE_CASE       :	Any						    =							output_from_past[:, :, random_slice_idx]

            # test that outputs are equal for slice
            tf.debugging.assert_near(__a    ,						__a    ,						rtol=1E-3							)





@require_tf
class 			lowerCAmelCase__( __lowercase  ,     __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    (TFOPTModel, TFOPTForCausalLM) if is_tf_available() else ()
       __snake_case 			=    (TFOPTForCausalLM,) if is_tf_available() else ()
       __snake_case 			=    (
           {'''feature-extraction''': TFOPTModel, '''text-generation''': TFOPTForCausalLM} if is_tf_available() else {}
       )
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    False
       __snake_case 			=    1_0




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							TFOPTModelTester(self							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							ConfigTester(self    ,						config_class=__a							)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            self.config_tester.run_common_tests()




       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							self.model_tester.prepare_config_and_inputs_for_common()
            self.model_tester.check_decoder_model_past_large_inputs(*__a							)







       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.model_tester.prepare_config_and_inputs_for_common()

            def _get_word_embedding_weight(__lowerCamelCase    ,						__lowerCamelCase							):
                 if hasattr(__a    ,						"weight"							):
                      return embedding_layer.weight
                 else:
                      # Here we build the word embeddings weights if not exists.
                      # And then we retry to get the attribute once built.
                      model.build()
                      if hasattr(__a    ,						"weight"							):
                           return embedding_layer.weight
                      else:
                           return None

            for model_class in self.all_model_classes:
                 for size in [config.vocab_size - 1_0, config.vocab_size + 1_0]:
                      # build the embeddings
                      _SCREAMING_SNAKE_CASE       :	List[str]						    =							model_class(config=__a							)
                      _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							_get_word_embedding_weight(__a    ,						model.get_input_embeddings()							)
                      _SCREAMING_SNAKE_CASE       :	int						    =							_get_word_embedding_weight(__a    ,						model.get_output_embeddings()							)

                      # reshape the embeddings
                      model.resize_token_embeddings(__a							)
                      _SCREAMING_SNAKE_CASE       :	List[Any]						    =							_get_word_embedding_weight(__a    ,						model.get_input_embeddings()							)
                      _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							_get_word_embedding_weight(__a    ,						model.get_output_embeddings()							)

                      # check that the resized embeddings size matches the desired size.
                      _SCREAMING_SNAKE_CASE       :	Tuple						    =							size if size is not None else config.vocab_size

                      self.assertEqual(new_input_embeddings.shape[0]    ,						__a							)

                      # check that weights remain the same after resizing
                      _SCREAMING_SNAKE_CASE       :	List[Any]						    =							True
                      for pa, pa in zip(old_input_embeddings.value()    ,						new_input_embeddings.value()							):
                           if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
                                _SCREAMING_SNAKE_CASE       :	Dict						    =							False
                      self.assertTrue(__a							)

                      if old_output_embeddings is not None and new_output_embeddings is not None:
                           self.assertEqual(new_output_embeddings.shape[0]    ,						__a							)

                           _SCREAMING_SNAKE_CASE       :	Any						    =							True
                           for pa, pa in zip(old_output_embeddings.value()    ,						new_output_embeddings.value()							):
                                if tf.math.reduce_sum(tf.math.abs(pa - pa							)							) > 0:
                                     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							False
                           self.assertTrue(__a							)






def     lowerCamelCase__			(__lowerCamelCase	):
     return tf.constant(UpperCamelCase__,       dtype=tf.intaa	)





@require_tf
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    9_9




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tf.ones((4, 1)    ,						dtype=tf.intaa							) * 2
            _SCREAMING_SNAKE_CASE       :	Any						    =							tf.concat([ids_tensor((4, 6)    ,						self.vocab_size - 3							) + 3, eos_column_vector]    ,						axis=1							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							input_ids.shape[0]
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							OPTConfig(
                vocab_size=self.vocab_size    ,						hidden_size=2_4    ,						num_hidden_layers=2    ,						num_attention_heads=2    ,						ffn_dim=3_2    ,						max_position_embeddings=4_8    ,						eos_token_id=2    ,						pad_token_id=1    ,						bos_token_id=0    ,						)
            return config, input_ids, batch_size





@require_sentencepiece
@require_tf
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       @slow
       def     UpperCamelCase_			(      self							)			->     int:
            _SCREAMING_SNAKE_CASE       :	int						    =							TFOPTModel.from_pretrained("facebook/opt-350m"							)
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							_long_tensor([[0, 3_1_4_1_4, 2_3_2, 3_2_8, 7_4_0, 1_1_4_0, 1_2_6_9_5, 6_9, 4_6_0_7_8, 1_5_8_8, 2]]							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tf.not_equal(__a    ,						model.config.pad_token_id							)
            with tf.GradientTape():
                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							model(input_ids=__a    ,						attention_mask=__a							).last_hidden_state
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							(1, 1_1, 5_1_2)
            self.assertEqual(output.shape    ,						__a							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tf.constant(
                [[-0.2873, -1.9218, -0.3033], [-1.2710, -0.1338, -0.1902], [0.4095, 0.1214, -1.3121]]							)
            self.assertTrue(np.allclose(output[:, :3, :3]    ,						__a    ,						atol=4E-3							)							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tf.function(__a    ,						jit_compile=__a							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							xla_generate(__a    ,						__a							)[0]
            self.assertTrue(np.allclose(output[:, :3, :3]    ,						__a    ,						atol=4E-2							)							)





@require_tf
@slow
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     str:
            super().setUp()
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							"""facebook/opt-350m"""







       def     UpperCamelCase_			(      self							)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							TFOPTForCausalLM.from_pretrained(self.path_model							)
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							GPTaTokenizer.from_pretrained(self.path_model							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							[
                """Today is a beautiful day and I want to""",
                """In the city of""",
                """Paris is the capital of France and""",
                """Computers and mobile phones have taken""",
            ]
            # verify that prompt without BOS token is identical to Metaseq -> add_special_tokens=False
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer(__a    ,						return_tensors="tf"    ,						padding=__a    ,						add_special_tokens=__a							)
            _SCREAMING_SNAKE_CASE       :	int						    =							tf.math.reduce_mean(model(inputs.input_ids    ,						attention_mask=inputs.attention_mask							)[0]    ,						axis=-1							)
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tf.constant(
                [
                    [1.3851, -13.8923, -10.5229, -10.7533, -0.2309, -10.2384, -0.5365, -9.0947, -5.1670],
                    [-4.7073, -10.6276, -3.9415, -21.5242, -0.2822, -0.2822, -0.2822, -0.2822, -0.2822],
                    [0.6247, -3.4229, -8.9179, -1.4297, -14.1650, 1.4146, -9.0218, -0.2703, -0.2703],
                    [6.4783, -1.9913, -10.7926, -2.3336, 1.5092, -0.9974, -6.8213, 1.3477, 1.3477],
                ]							)
            self.assertTrue(np.allclose(__a    ,						__a    ,						atol=1E-4							)							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							tf.function(__a    ,						jit_compile=__a							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tf.math.reduce_mean(xla_generate(inputs.input_ids    ,						attention_mask=inputs.attention_mask							)[0]    ,						axis=-1							)
            self.assertTrue(np.allclose(__a    ,						__a    ,						atol=1E-4							)							)





@require_tf
@slow
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       @property
       def     UpperCamelCase_			(      self							)			->     int:
            return [
                "Today is a beautiful day and I want",
                "In the city of",
                "Paris is the capital of France and",
                "Computers and mobile phones have taken",
            ]




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	int						    =							"""facebook/opt-125m"""

            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							[
                """Today is a beautiful day and I want to""",
                """In the city of New York, the city""",
                """Paris is the capital of France and the capital""",
                """Computers and mobile phones have taken over the""",
            ]

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							[]
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							GPTaTokenizer.from_pretrained(__a							)
            _SCREAMING_SNAKE_CASE       :	str						    =							TFOPTForCausalLM.from_pretrained(__a							)

            for prompt in self.prompts:
                 _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer(__a    ,						return_tensors="tf"							).input_ids

                 _SCREAMING_SNAKE_CASE       :	Any						    =							model.generate(__a    ,						max_length=1_0							)

                 _SCREAMING_SNAKE_CASE       :	Tuple						    =							tokenizer.batch_decode(__a    ,						skip_special_tokens=__a							)
                 predicted_outputs += generated_string

            self.assertListEqual(__a    ,						__a							)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							"""facebook/opt-350m"""

            _SCREAMING_SNAKE_CASE       :	str						    =							GPTaTokenizer.from_pretrained(__a							)
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							TFOPTForCausalLM.from_pretrained(__a							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"""left"""

            # use different length sentences to test batching
            _SCREAMING_SNAKE_CASE       :	Any						    =							[
                """Hello, my dog is a little""",
                """Today, I""",
            ]

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenizer(__a    ,						return_tensors="tf"    ,						padding=__a							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							inputs["""input_ids"""]

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							model.generate(input_ids=__a    ,						attention_mask=inputs["attention_mask"]							)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							tokenizer(sentences[0]    ,						return_tensors="tf"							).input_ids
            _SCREAMING_SNAKE_CASE       :	Dict						    =							model.generate(input_ids=__a							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							inputs_non_padded.shape[-1] - tf.math.reduce_sum(
                tf.cast(inputs["attention_mask"][-1]    ,						tf.intaa							)							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer(sentences[1]    ,						return_tensors="tf"							).input_ids
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							model.generate(input_ids=__a    ,						max_length=model.config.max_length - num_paddings							)

            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.batch_decode(__a    ,						skip_special_tokens=__a							)
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							tokenizer.decode(output_non_padded[0]    ,						skip_special_tokens=__a							)
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							tokenizer.decode(output_padded[0]    ,						skip_special_tokens=__a							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							[
                """Hello, my dog is a little bit of a dork.\nI'm a little bit""",
                """Today, I was in the middle of a conversation with a friend about the""",
            ]
            self.assertListEqual(__a    ,						__a							)
            self.assertListEqual(__a    ,						[non_padded_sentence, padded_sentence]							)







       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							"""facebook/opt-350m"""

            _SCREAMING_SNAKE_CASE       :	Dict						    =							[
                """Today is a beautiful day and I want to""",
                """In the city of San Francisco, the city""",
                """Paris is the capital of France and the capital""",
                """Computers and mobile phones have taken over the""",
            ]

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							[]
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							GPTaTokenizer.from_pretrained(__a							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							TFOPTForCausalLM.from_pretrained(__a							)

            for prompt in self.prompts:
                 _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer(__a    ,						return_tensors="tf"							).input_ids

                 _SCREAMING_SNAKE_CASE       :	Dict						    =							model.generate(__a    ,						max_length=1_0							)

                 _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.batch_decode(__a    ,						skip_special_tokens=__a							)
                 predicted_outputs += generated_string

            self.assertListEqual(__a    ,						__a							)






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False
       __snake_case 			=    True




       def     UpperCamelCase_			(      self							)			->     Dict:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	Dict						    =							[
                "[UNK]",
                "[CLS]",
                "[SEP]",
                "こんにちは",
                "こん",
                "にちは",
                "ばんは",
                "##こん",
                "##にちは",
                "##ばんは",
                "世界",
                "##世界",
                "、",
                "##、",
                "。",
                "##。",
            ]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	str						    =							"こんにちは 、 世界 。 こんばんは 、 世界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Dict:
            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	int						    =							self.get_input_output_texts(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode(__lowerCamelCase    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.decode(__lowerCamelCase    ,						clean_up_tokenization_spaces=__lowerCamelCase							)
            return text, ids




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							self.tokenizer_class(self.vocab_file							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="mecab"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     str:
            try:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							MecabTokenizer(mecab_dic="unidic_lite"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            try:
                 _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(mecab_dic="unidic"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(do_lower_case=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Dict:
            try:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(
                     do_lower_case=__lowerCamelCase    ,						normalize_text=__lowerCamelCase    ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic"							)
            except RuntimeError:
                 # if dict doesn't exist in the system, previous code raises this error.
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							MecabTokenizer(normalize_text=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="sudachi"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="A"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国", "人", "参政", "権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     int:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="B"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人", "参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="C"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							SudachiTokenizer(do_lower_case=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							SudachiTokenizer(normalize_text=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							SudachiTokenizer(trim_whitespace=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="jumanpp"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							JumanppTokenizer(do_lower_case=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							JumanppTokenizer(normalize_text=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							JumanppTokenizer(trim_whitespace=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"							)    ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	int						    =							["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

            _SCREAMING_SNAKE_CASE       :	str						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							i
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							WordpieceTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こんにちは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは"							)    ,						["こん", "##ばんは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"							)    ,						["こん", "##ばんは", "[UNK]", "こんにちは"]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	int						    =							BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.subword_tokenizer

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]							)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]							)







       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						**__lowerCamelCase							)			->     List[Any]:
            return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						subword_tokenizer_type="character"    ,						**__lowerCamelCase							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self							)			->     Tuple:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     int:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	int						    =							self.tokenizer_class(self.vocab_file    ,						subword_tokenizer_type="character"							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"							)
            self.assertListEqual(
                __lowerCamelCase    ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]							)
            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	int						    =							i
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							CharacterTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こ", "ん", "に", "ち", "は"]							)

            self.assertListEqual(tokenizer.tokenize("こんにちほ"							)    ,						["こ", "ん", "に", "ち", "[UNK]"]							)







       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							"cl-tohoku/bert-base-japanese"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							AutoTokenizer.from_pretrained(__lowerCamelCase							)
            self.assertIsInstance(__lowerCamelCase    ,						__lowerCamelCase							)





class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"cl-tohoku/bert-base-japanese"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"bert-base-cased"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertJapaneseTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)