





import os
from pickle import UnpicklingError
from typing import Dict, Tuple

import jax
import jax.numpy as jnp
import numpy as np
from flax.serialization import from_bytes
from flax.traverse_util import flatten_dict, unflatten_dict

import transformers

from .utils import logging


UpperCamelCase__      =logging.get_logger(__name__)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase=False	):
     try:
          import torch  # noqa: F401
     except ImportError:
          logger.error(
              "Loading a PyTorch model in Flax, requires both PyTorch and Flax to be installed. Please see"
              " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"
              " instructions."	)
          raise

     if not is_sharded:
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							os.path.abspath(__lowerCamelCase	)
          logger.info(f"""Loading PyTorch weights from {pt_path}"""	)

          _SCREAMING_SNAKE_CASE       :	Any						    =							torch.load(__lowerCamelCase,       map_location="cpu"	)
          logger.info(f"""PyTorch checkpoint contains {sum(t.numel() for t in pt_state_dict.values()	):,} parameters."""	)

          _SCREAMING_SNAKE_CASE       :	Any						    =							convert_pytorch_state_dict_to_flax(__lowerCamelCase,       __lowerCamelCase	)
     else:
          # model is sharded and pytorch_checkpoint_path already contains the list of .pt shard files
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							convert_pytorch_sharded_state_dict_to_flax(__lowerCamelCase,       __lowerCamelCase	)
     return flax_state_dict






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       ):

     def is_key_or_prefix_key_in_dict(__lowerCamelCase	) -> bool:
          return len(set(__lowerCamelCase	) & {key, (model_prefix,) + key}	) > 0

     # layer norm
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							pt_tuple_key[:-1] + ("scale",)
     if pt_tuple_key[-1] in ["weight", "gamma"] and is_key_or_prefix_key_in_dict(__lowerCamelCase	):
          return renamed_pt_tuple_key, pt_tensor

     # batch norm layer mean
     _SCREAMING_SNAKE_CASE       :	int						    =							pt_tuple_key[:-1] + ("mean",)
     if pt_tuple_key[-1] == "running_mean" and not is_key_or_prefix_key_in_dict(__lowerCamelCase	):
          return renamed_pt_tuple_key, pt_tensor

     # batch norm layer var
     _SCREAMING_SNAKE_CASE       :	Dict						    =							pt_tuple_key[:-1] + ("var",)
     if pt_tuple_key[-1] == "running_var" and not is_key_or_prefix_key_in_dict(__lowerCamelCase	):
          return renamed_pt_tuple_key, pt_tensor

     # embedding
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							pt_tuple_key[:-1] + ("embedding",)
     if pt_tuple_key[-1] == "weight" and is_key_or_prefix_key_in_dict(__lowerCamelCase	):
          return renamed_pt_tuple_key, pt_tensor

     # conv layer
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							pt_tuple_key[:-1] + ("kernel",)
     if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4 and not is_key_or_prefix_key_in_dict(__lowerCamelCase	):
          _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							pt_tensor.transpose(2,       3,       1,       0	)
          return renamed_pt_tuple_key, pt_tensor

     # linear layer
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							pt_tuple_key[:-1] + ("kernel",)
     if pt_tuple_key[-1] == "weight" and not is_key_or_prefix_key_in_dict(__lowerCamelCase	):
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							pt_tensor.T
          return renamed_pt_tuple_key, pt_tensor

     # old PyTorch layer norm weight
     _SCREAMING_SNAKE_CASE       :	Dict						    =							pt_tuple_key[:-1] + ("weight",)
     if pt_tuple_key[-1] == "gamma":
          return renamed_pt_tuple_key, pt_tensor

     # old PyTorch layer norm bias
     _SCREAMING_SNAKE_CASE       :	Dict						    =							pt_tuple_key[:-1] + ("bias",)
     if pt_tuple_key[-1] == "beta":
          return renamed_pt_tuple_key, pt_tensor

     # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							None
     if pt_tuple_key[-3::2] == ("parametrizations", "original0"):
          _SCREAMING_SNAKE_CASE       :	List[str]						    =							pt_tuple_key[-2] + "_g"
     elif pt_tuple_key[-3::2] == ("parametrizations", "original1"):
          _SCREAMING_SNAKE_CASE       :	int						    =							pt_tuple_key[-2] + "_v"
     if name is not None:
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							pt_tuple_key[:-3] + (name,)
          return renamed_pt_tuple_key, pt_tensor

     return pt_tuple_key, pt_tensor






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     # convert pytorch tensor to numpy
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							{k: v.numpy() for k, v in pt_state_dict.items()}

     _SCREAMING_SNAKE_CASE       :	Tuple						    =							flax_model.base_model_prefix

     # use params dict if the model contains batch norm layers
     if "params" in flax_model.params:
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							flax_model.params["params"]
     else:
          _SCREAMING_SNAKE_CASE       :	int						    =							flax_model.params
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							flatten_dict(__lowerCamelCase	)

     # add batch_stats keys,values to dict
     if "batch_stats" in flax_model.params:
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							flatten_dict(flax_model.params["batch_stats"]	)
          random_flax_state_dict.update(__lowerCamelCase	)

     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							{}

     _SCREAMING_SNAKE_CASE       :	Any						    =							(model_prefix not in flax_model_params) and (
         model_prefix in {k.split("."	)[0] for k in pt_state_dict.keys()}
     )
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							(model_prefix in flax_model_params) and (
         model_prefix not in {k.split("."	)[0] for k in pt_state_dict.keys()}
     )

     # Need to change some parameters name to match Flax names
     for pt_key, pt_tensor in pt_state_dict.items():
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							tuple(pt_key.split("."	)	)

          # remove base model prefix if necessary
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							pt_tuple_key[0] == model_prefix
          if load_model_with_head_into_base_model and has_base_model_prefix:
               _SCREAMING_SNAKE_CASE       :	int						    =							pt_tuple_key[1:]

          # Correctly rename weight parameters
          _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Tuple						    =							rename_key_and_reshape_tensor(
              __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)

          # add model prefix if necessary
          _SCREAMING_SNAKE_CASE       :	Dict						    =							(model_prefix,) + flax_key in random_flax_state_dict
          if load_base_model_into_model_with_head and require_base_model_prefix:
               _SCREAMING_SNAKE_CASE       :	int						    =							(model_prefix,) + flax_key

          if flax_key in random_flax_state_dict:
               if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                    raise ValueError(
                        f"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
                        f"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)

        # add batch stats if the model contains batchnorm layers
          if "batch_stats" in flax_model.params:
               if "mean" in flax_key[-1] or "var" in flax_key[-1]:
                    _SCREAMING_SNAKE_CASE       :	str						    =							jnp.asarray(__lowerCamelCase	)
                    continue
               # remove num_batches_tracked key
               if "num_batches_tracked" in flax_key[-1]:
                    flax_state_dict.pop(__lowerCamelCase,       __lowerCamelCase	)
                    continue

               # also add unexpected weight so that warning is thrown
               _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							jnp.asarray(__lowerCamelCase	)

          else:
               # also add unexpected weight so that warning is thrown
               _SCREAMING_SNAKE_CASE       :	int						    =							jnp.asarray(__lowerCamelCase	)

     return unflatten_dict(__lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     import torch

     # Load the index
     _SCREAMING_SNAKE_CASE       :	Dict						    =							{}
     for shard_file in shard_filenames:
          # load using msgpack utils
          _SCREAMING_SNAKE_CASE       :	Any						    =							torch.load(__lowerCamelCase	)
          _SCREAMING_SNAKE_CASE       :	str						    =							{k: v.numpy() for k, v in pt_state_dict.items()}

          _SCREAMING_SNAKE_CASE       :	Any						    =							flax_model.base_model_prefix

          # use params dict if the model contains batch norm layers and then add batch_stats keys,values to dict
          if "batch_stats" in flax_model.params:
               _SCREAMING_SNAKE_CASE       :	str						    =							flax_model.params["params"]

               _SCREAMING_SNAKE_CASE       :	Tuple						    =							flatten_dict(__lowerCamelCase	)
               random_flax_state_dict.update(flatten_dict(flax_model.params["batch_stats"]	)	)
          else:
               _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							flax_model.params
               _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							flatten_dict(__lowerCamelCase	)

          _SCREAMING_SNAKE_CASE       :	int						    =							(model_prefix not in flax_model_params) and (
              model_prefix in {k.split("."	)[0] for k in pt_state_dict.keys()}
          )
          _SCREAMING_SNAKE_CASE       :	List[Any]						    =							(model_prefix in flax_model_params) and (
              model_prefix not in {k.split("."	)[0] for k in pt_state_dict.keys()}
          )
          # Need to change some parameters name to match Flax names
          for pt_key, pt_tensor in pt_state_dict.items():
               _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							tuple(pt_key.split("."	)	)

               # remove base model prefix if necessary
               _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							pt_tuple_key[0] == model_prefix
               if load_model_with_head_into_base_model and has_base_model_prefix:
                    _SCREAMING_SNAKE_CASE       :	List[Any]						    =							pt_tuple_key[1:]

               # Correctly rename weight parameters
               _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Tuple						    =							rename_key_and_reshape_tensor(
                   __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)
               # add model prefix if necessary
               _SCREAMING_SNAKE_CASE       :	int						    =							(model_prefix,) + flax_key in random_flax_state_dict
               if load_base_model_into_model_with_head and require_base_model_prefix:
                    _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							(model_prefix,) + flax_key

               if flax_key in random_flax_state_dict:
                    if flax_tensor.shape != random_flax_state_dict[flax_key].shape:
                         raise ValueError(
                             f"""PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape """
                             f"""{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)

            # add batch stats if the model contains batchnorm layers
               if "batch_stats" in flax_model.params:
                    if "mean" in flax_key[-1]:
                         _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							jnp.asarray(__lowerCamelCase	)
                         continue
                    if "var" in flax_key[-1]:
                         _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							jnp.asarray(__lowerCamelCase	)
                         continue
                    # remove num_batches_tracked key
                    if "num_batches_tracked" in flax_key[-1]:
                         flax_state_dict.pop(__lowerCamelCase,       __lowerCamelCase	)
                         continue

                    # also add unexpected weight so that warning is thrown
                    _SCREAMING_SNAKE_CASE       :	int						    =							jnp.asarray(__lowerCamelCase	)

               else:
                    # also add unexpected weight so that warning is thrown
                    _SCREAMING_SNAKE_CASE       :	List[Any]						    =							jnp.asarray(__lowerCamelCase	)
     return unflatten_dict(__lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):
     _SCREAMING_SNAKE_CASE       :	Dict						    =							os.path.abspath(__lowerCamelCase	)
     logger.info(f"""Loading Flax weights from {flax_checkpoint_path}"""	)

     # import correct flax class
     _SCREAMING_SNAKE_CASE       :	Tuple						    =							getattr(__lowerCamelCase,       "Flax" + model.__class__.__name__	)

     # load flax weight dict
     with open(__lowerCamelCase,       "rb"	) as state_f:
          try:
               _SCREAMING_SNAKE_CASE       :	Dict						    =							from_bytes(__lowerCamelCase,       state_f.read()	)
          except UnpicklingError:
               raise EnvironmentError(f"""Unable to convert {flax_checkpoint_path} to Flax deserializable object. """	)

     return load_flax_weights_in_pytorch_model(__lowerCamelCase,       __lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase	):

     try:
          import torch  # noqa: F401
     except ImportError:
          logger.error(
              "Loading a Flax weights in PyTorch, requires both PyTorch and Flax to be installed. Please see"
              " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation"
              " instructions."	)
          raise

     # check if we have bf16 weights
     _SCREAMING_SNAKE_CASE       :	Any						    =							flatten_dict(jax.tree_util.tree_map(lambda __lowerCamelCase	: x.dtype == jnp.bfloataa,       __lowerCamelCase	)	).values()
     if any(__lowerCamelCase	):
          # convert all weights to fp32 if the are bf16 since torch.from_numpy can-not handle bf16
          # and bf16 is not fully supported in PT yet.
          logger.warning(
              "Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` "
              "before loading those in PyTorch model."	)
          _SCREAMING_SNAKE_CASE       :	int						    =							jax.tree_util.tree_map(
              lambda __lowerCamelCase	: params.astype(np.floataa	) if params.dtype == jnp.bfloataa else params,       __lowerCamelCase	)

     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							flatten_dict(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Dict						    =							pt_model.state_dict()

     _SCREAMING_SNAKE_CASE       :	Any						    =							(pt_model.base_model_prefix in flax_state) and (
         pt_model.base_model_prefix not in {k.split("."	)[0] for k in pt_model_dict.keys()}
     )
     _SCREAMING_SNAKE_CASE       :	int						    =							(pt_model.base_model_prefix not in flax_state) and (
         pt_model.base_model_prefix in {k.split("."	)[0] for k in pt_model_dict.keys()}
     )

     # keep track of unexpected & missing keys
     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							[]
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							set(pt_model_dict.keys()	)

     for flax_key_tuple, flax_tensor in flax_state_dict.items():
          _SCREAMING_SNAKE_CASE       :	Dict						    =							flax_key_tuple[0] == pt_model.base_model_prefix
          _SCREAMING_SNAKE_CASE       :	str						    =							".".join((pt_model.base_model_prefix,) + flax_key_tuple	) in pt_model_dict

          # adapt flax_key to prepare for loading from/to base model only
          if load_model_with_head_into_base_model and has_base_model_prefix:
               _SCREAMING_SNAKE_CASE       :	List[str]						    =							flax_key_tuple[1:]
          elif load_base_model_into_model_with_head and require_base_model_prefix:
               _SCREAMING_SNAKE_CASE       :	Dict						    =							(pt_model.base_model_prefix,) + flax_key_tuple

          # rename flax weights to PyTorch format
          if flax_key_tuple[-1] == "kernel" and flax_tensor.ndim == 4 and ".".join(__lowerCamelCase	) not in pt_model_dict:
               # conv layer
               _SCREAMING_SNAKE_CASE       :	str						    =							flax_key_tuple[:-1] + ("weight",)
               _SCREAMING_SNAKE_CASE       :	str						    =							jnp.transpose(__lowerCamelCase,       (3, 2, 0, 1)	)
          elif flax_key_tuple[-1] == "kernel" and ".".join(__lowerCamelCase	) not in pt_model_dict:
               # linear layer
               _SCREAMING_SNAKE_CASE       :	List[str]						    =							flax_key_tuple[:-1] + ("weight",)
               _SCREAMING_SNAKE_CASE       :	List[Any]						    =							flax_tensor.T
          elif flax_key_tuple[-1] in ["scale", "embedding"]:
               _SCREAMING_SNAKE_CASE       :	Tuple						    =							flax_key_tuple[:-1] + ("weight",)

          # adding batch stats from flax batch norm to pt
          elif "mean" in flax_key_tuple[-1]:
               _SCREAMING_SNAKE_CASE       :	List[str]						    =							flax_key_tuple[:-1] + ("running_mean",)
          elif "var" in flax_key_tuple[-1]:
               _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							flax_key_tuple[:-1] + ("running_var",)

          if "batch_stats" in flax_state:
               _SCREAMING_SNAKE_CASE       :	Dict						    =							".".join(flax_key_tuple[1:]	)  # Remove the params/batch_stats header
          else:
               _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							".".join(__lowerCamelCase	)

          # We also need to look at `pt_model_dict` and see if there are keys requiring further transformation.
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							{}
          # New `weight_norm` from https://github.com/huggingface/transformers/pull/24030
          for key in pt_model_dict:
               _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							key.split("."	)
               _SCREAMING_SNAKE_CASE       :	List[str]						    =							None
               if key_components[-3::2] == ["parametrizations", "original0"]:
                    _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							key_components[-2] + "_g"
               elif key_components[-3::2] == ["parametrizations", "original1"]:
                    _SCREAMING_SNAKE_CASE       :	Dict						    =							key_components[-2] + "_v"
               if name is not None:
                    _SCREAMING_SNAKE_CASE       :	Dict						    =							key_components[:-3] + [name]
                    _SCREAMING_SNAKE_CASE       :	Dict						    =							".".join(__lowerCamelCase	)
                    _SCREAMING_SNAKE_CASE       :	List[Any]						    =							key

          if flax_key in special_pt_names:
               _SCREAMING_SNAKE_CASE       :	List[Any]						    =							special_pt_names[flax_key]

          if flax_key in pt_model_dict:
               if flax_tensor.shape != pt_model_dict[flax_key].shape:
                    raise ValueError(
                        f"""Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected """
                        f"""to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}."""	)
               else:
                    # add weight to pytorch dict
                    _SCREAMING_SNAKE_CASE       :	str						    =							np.asarray(__lowerCamelCase	) if not isinstance(__lowerCamelCase,       np.ndarray	) else flax_tensor
                    _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							torch.from_numpy(__lowerCamelCase	)
                    # remove from missing keys
                    missing_keys.remove(__lowerCamelCase	)
          else:
               # weight is not expected by PyTorch model
               unexpected_keys.append(__lowerCamelCase	)

     pt_model.load_state_dict(__lowerCamelCase	)

     # re-transform missing_keys to list
     _SCREAMING_SNAKE_CASE       :	Dict						    =							list(__lowerCamelCase	)

     if len(__lowerCamelCase	) > 0:
          logger.warning(
              "Some weights of the Flax model were not used when initializing the PyTorch model"
              f""" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing"""
              f""" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture"""
              " (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This"
              f""" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect"""
              " to be exactly identical (e.g. initializing a BertForSequenceClassification model from a"
              " FlaxBertForSequenceClassification model)."	)
     else:
          logger.warning(f"""All Flax model weights were used when initializing {pt_model.__class__.__name__}.\n"""	)
     if len(__lowerCamelCase	) > 0:
          logger.warning(
              f"""Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly"""
              f""" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to"""
              " use it for predictions and inference."	)
     else:
          logger.warning(
              f"""All the weights of {pt_model.__class__.__name__} were initialized from the Flax model.\n"""
              "If your task is similar to the task the model of the checkpoint was trained on, "
              f"""you can already use {pt_model.__class__.__name__} for predictions without further training."""	)

     return pt_model






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False
       __snake_case 			=    True




       def     UpperCamelCase_			(      self							)			->     Dict:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	Dict						    =							[
                "[UNK]",
                "[CLS]",
                "[SEP]",
                "こんにちは",
                "こん",
                "にちは",
                "ばんは",
                "##こん",
                "##にちは",
                "##ばんは",
                "世界",
                "##世界",
                "、",
                "##、",
                "。",
                "##。",
            ]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	str						    =							"こんにちは 、 世界 。 こんばんは 、 世界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Dict:
            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	int						    =							self.get_input_output_texts(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode(__lowerCamelCase    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.decode(__lowerCamelCase    ,						clean_up_tokenization_spaces=__lowerCamelCase							)
            return text, ids




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							self.tokenizer_class(self.vocab_file							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="mecab"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     str:
            try:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							MecabTokenizer(mecab_dic="unidic_lite"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            try:
                 _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(mecab_dic="unidic"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(do_lower_case=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Dict:
            try:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(
                     do_lower_case=__lowerCamelCase    ,						normalize_text=__lowerCamelCase    ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic"							)
            except RuntimeError:
                 # if dict doesn't exist in the system, previous code raises this error.
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							MecabTokenizer(normalize_text=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="sudachi"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="A"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国", "人", "参政", "権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     int:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="B"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人", "参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="C"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							SudachiTokenizer(do_lower_case=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							SudachiTokenizer(normalize_text=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							SudachiTokenizer(trim_whitespace=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="jumanpp"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							JumanppTokenizer(do_lower_case=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							JumanppTokenizer(normalize_text=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							JumanppTokenizer(trim_whitespace=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"							)    ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	int						    =							["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

            _SCREAMING_SNAKE_CASE       :	str						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							i
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							WordpieceTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こんにちは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは"							)    ,						["こん", "##ばんは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"							)    ,						["こん", "##ばんは", "[UNK]", "こんにちは"]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	int						    =							BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.subword_tokenizer

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]							)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]							)







       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						**__lowerCamelCase							)			->     List[Any]:
            return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						subword_tokenizer_type="character"    ,						**__lowerCamelCase							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self							)			->     Tuple:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     int:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	int						    =							self.tokenizer_class(self.vocab_file    ,						subword_tokenizer_type="character"							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"							)
            self.assertListEqual(
                __lowerCamelCase    ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]							)
            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	int						    =							i
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							CharacterTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こ", "ん", "に", "ち", "は"]							)

            self.assertListEqual(tokenizer.tokenize("こんにちほ"							)    ,						["こ", "ん", "に", "ち", "[UNK]"]							)







       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							"cl-tohoku/bert-base-japanese"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							AutoTokenizer.from_pretrained(__lowerCamelCase							)
            self.assertIsInstance(__lowerCamelCase    ,						__lowerCamelCase							)





class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"cl-tohoku/bert-base-japanese"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"bert-base-cased"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertJapaneseTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)