



'''simple docstring'''

# Copyright 2021 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from packaging import version

from .. import __version__
from .constants import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_STANDARD_MEAN, IMAGENET_STANDARD_STD
from .doc import (
    add_code_sample_docstrings,
    add_end_docstrings,
    add_start_docstrings,
    add_start_docstrings_to_model_forward,
    copy_func,
    replace_return_docstrings,
)
from .generic import (
    ContextManagers,
    ExplicitEnum,
    ModelOutput,
    PaddingStrategy,
    TensorType,
    add_model_info_to_auto_map,
    cached_property,
    can_return_loss,
    expand_dims,
    find_labels,
    flatten_dict,
    infer_framework,
    is_jax_tensor,
    is_numpy_array,
    is_tensor,
    is_tf_symbolic_tensor,
    is_tf_tensor,
    is_torch_device,
    is_torch_dtype,
    is_torch_tensor,
    reshape,
    squeeze,
    strtobool,
    tensor_size,
    to_numpy,
    to_py_obj,
    transpose,
    working_or_temp_dir,
)
from .hub import (
    CLOUDFRONT_DISTRIB_PREFIX,
    DISABLE_TELEMETRY,
    HF_MODULES_CACHE,
    HUGGINGFACE_CO_PREFIX,
    HUGGINGFACE_CO_RESOLVE_ENDPOINT,
    PYTORCH_PRETRAINED_BERT_CACHE,
    PYTORCH_TRANSFORMERS_CACHE,
    S3_BUCKET_PREFIX,
    TRANSFORMERS_CACHE,
    TRANSFORMERS_DYNAMIC_MODULE_NAME,
    EntryNotFoundError,
    PushToHubMixin,
    RepositoryNotFoundError,
    RevisionNotFoundError,
    cached_file,
    default_cache_path,
    define_sagemaker_information,
    download_url,
    extract_commit_hash,
    get_cached_models,
    get_file_from_repo,
    get_full_repo_name,
    has_file,
    http_user_agent,
    is_offline_mode,
    is_remote_url,
    move_cache,
    send_example_telemetry,
    try_to_load_from_cache,
)
from .import_utils import (
    ENV_VARS_TRUE_AND_AUTO_VALUES,
    ENV_VARS_TRUE_VALUES,
    TORCH_FX_REQUIRED_VERSION,
    USE_JAX,
    USE_TF,
    USE_TORCH,
    DummyObject,
    OptionalDependencyNotAvailable,
    _LazyModule,
    ccl_version,
    direct_transformers_import,
    get_torch_version,
    is_accelerate_available,
    is_apex_available,
    is_bitsandbytes_available,
    is_bsa_available,
    is_coloredlogs_available,
    is_cython_available,
    is_datasets_available,
    is_decord_available,
    is_detectrona_available,
    is_faiss_available,
    is_flax_available,
    is_ftfy_available,
    is_in_notebook,
    is_ipex_available,
    is_jieba_available,
    is_jumanpp_available,
    is_kenlm_available,
    is_keras_nlp_available,
    is_librosa_available,
    is_natten_available,
    is_ninja_available,
    is_onnx_available,
    is_openai_available,
    is_optimum_available,
    is_pandas_available,
    is_peft_available,
    is_phonemizer_available,
    is_protobuf_available,
    is_psutil_available,
    is_pyanvml_available,
    is_pyctcdecode_available,
    is_pytesseract_available,
    is_pytest_available,
    is_pytorch_quantization_available,
    is_rjieba_available,
    is_sacremoses_available,
    is_safetensors_available,
    is_sagemaker_dp_enabled,
    is_sagemaker_mp_enabled,
    is_scipy_available,
    is_sentencepiece_available,
    is_seqio_available,
    is_sklearn_available,
    is_soundfile_availble,
    is_spacy_available,
    is_speech_available,
    is_sudachi_available,
    is_tensorflow_probability_available,
    is_tensorflow_text_available,
    is_tfaonnx_available,
    is_tf_available,
    is_timm_available,
    is_tokenizers_available,
    is_torch_available,
    is_torch_bfaa_available,
    is_torch_bfaa_cpu_available,
    is_torch_bfaa_gpu_available,
    is_torch_compile_available,
    is_torch_cuda_available,
    is_torch_fx_available,
    is_torch_fx_proxy,
    is_torch_mps_available,
    is_torch_neuroncore_available,
    is_torch_tensorrt_fx_available,
    is_torch_tfaa_available,
    is_torch_tpu_available,
    is_torchaudio_available,
    is_torchdistx_available,
    is_torchdynamo_available,
    is_torchvision_available,
    is_training_run_on_sagemaker,
    is_vision_available,
    requires_backends,
    torch_only_method,
)


SCREAMING_SNAKE_CASE__     :						Dict	=  '''pytorch_model.bin'''
SCREAMING_SNAKE_CASE__     :						Dict	=  '''pytorch_model.bin.index.json'''
SCREAMING_SNAKE_CASE__     :						Any	=  '''adapter_config.json'''
SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  '''adapter_model.bin'''
SCREAMING_SNAKE_CASE__     :						Tuple	=  '''adapter_model.safetensors'''
SCREAMING_SNAKE_CASE__     :						Any	=  '''tf_model.h5'''
SCREAMING_SNAKE_CASE__     :						List[Any]	=  '''tf_model.h5.index.json'''
SCREAMING_SNAKE_CASE__     :						int	=  '''model.ckpt'''
SCREAMING_SNAKE_CASE__     :						List[str]	=  '''flax_model.msgpack'''
SCREAMING_SNAKE_CASE__     :						List[str]	=  '''flax_model.msgpack.index.json'''
SCREAMING_SNAKE_CASE__     :						List[str]	=  '''model.safetensors'''
SCREAMING_SNAKE_CASE__     :						List[Any]	=  '''model.safetensors.index.json'''
SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  '''config.json'''
SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  '''preprocessor_config.json'''
SCREAMING_SNAKE_CASE__     :						Tuple	=  FEATURE_EXTRACTOR_NAME
SCREAMING_SNAKE_CASE__     :						Any	=  '''generation_config.json'''
SCREAMING_SNAKE_CASE__     :						Any	=  '''modelcard.json'''

SCREAMING_SNAKE_CASE__     :						Any	=  '''▁'''
SCREAMING_SNAKE_CASE__     :						Union[str, Any]	=  SENTENCEPIECE_UNDERLINE  # Kept for backward compatibility

SCREAMING_SNAKE_CASE__     :						Optional[Any]	=  [
    [[0, 1, 0, 1], [1, 0, 0, 1]]
] * 2  # Needs to have 0s and 1s only since XLM uses it for langs too.
SCREAMING_SNAKE_CASE__     :						int	=  [[7, 6, 0, 0, 1], [1, 2, 3, 0, 0], [0, 0, 0, 4, 5]]
SCREAMING_SNAKE_CASE__     :						str	=  [[1, 1, 1, 1, 1], [1, 1, 1, 0, 0], [0, 0, 0, 1, 1]]






def 				a  (     UpperCamelCase_    :    Dict			)       ->	Dict:
		if version.parse(UpperCamelCase_			) < version.parse(UpperCamelCase_			):
				if "dev" in min_version:
						snake_case__	     =(
						    'This example requires a source install from HuggingFace Transformers (see '
						    '`https://huggingface.co/docs/transformers/installation#install-from-source`),'
						)
				else:
						snake_case__	     =f"""This example requires a minimum version of {min_version},"""
				error_message += f""" but the version found is {__version__}.\n"""
				raise ImportError(
				    error_message
				    + 'Check out https://github.com/huggingface/transformers/tree/main/examples#important-note for the examples corresponding to other '
				    'versions of HuggingFace Transformers.'			)









'''simple docstring'''

import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin

@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: str									=      BertJapaneseTokenizer
			a_	: List[str]									=      False
			a_	: List[Any]									=      True






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					super().setUp()

					snake_case__	     =[
					    '[UNK]',
					    '[CLS]',
					    '[SEP]',
					    'こんにちは',
					    'こん',
					    'にちは',
					    'ばんは',
					    '##こん',
					    '##にちは',
					    '##ばんは',
					    '世界',
					    '##世界',
					    '、',
					    '##、',
					    '。',
					    '##。',
					]

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							str:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こんにちは 、 世界 。 こんばんは 、 世界 。'
					return input_text, output_text






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Any:
					snake_case__     ,	snake_case__	     =self.get_input_output_texts(_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode(_UpperCAmelCase					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.decode(_UpperCAmelCase					,   clean_up_tokenization_spaces=_UpperCAmelCase						)
					return text, ids






			def 						_lowercase  (     self						)							->							Dict:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							List[str]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							str:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。\nこんばんは、世界。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='mecab'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =MecabTokenizer(mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							str:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic_lite'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[str]:
					try:
							snake_case__	     =MecabTokenizer(mecab_dic='unidic'						)
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =MecabTokenizer(do_lower_case=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップルストア', 'で', 'iphone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					try:
							snake_case__	     =MecabTokenizer(
							    do_lower_case=_UpperCAmelCase					,   normalize_text=_UpperCAmelCase					,   mecab_option='-d /usr/local/lib/mecab/dic/jumandic'						)
					except RuntimeError:
							# if dict doesn't exist in the system, previous code raises this error.
							return

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =MecabTokenizer(normalize_text=_UpperCAmelCase					,   mecab_dic='ipadic'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱｯﾌﾟﾙストア', 'で', 'iPhone', '８', 'が', '発売', 'さ', 'れ', 'た', '　', '。']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='sudachi'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Union[str, Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iPhone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='A'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国', '人', '参政', '権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							List[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='B'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人', '参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =SudachiTokenizer(sudachi_dict_type='core'					,   sudachi_split_mode='C'						)

					self.assertListEqual(tokenizer.tokenize('外国人参政権'						)					,   ['外国人参政権']						)






			@require_sudachi
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =SudachiTokenizer(do_lower_case=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'アップル', 'ストア', 'で', 'iphone', '8', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', ' ', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Dict:
					snake_case__	     =SudachiTokenizer(normalize_text=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   [' ', '\t', 'ｱｯﾌﾟﾙ', 'ストア', 'で', 'iPhone', '８', ' ', 'が', ' ', ' ', '\n ', '発売', 'さ', 'れ', 'た', '\u3000', '。', ' ', ' ']					,   )






			@require_sudachi
			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =SudachiTokenizer(trim_whitespace=_UpperCAmelCase					,   sudachi_dict_type='core'						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れ', 'た', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   word_tokenizer_type='jumanpp'						)
					self.assertIsNotNone(_UpperCAmelCase						)

					snake_case__	     ='こんにちは、世界。\nこんばんは、世界。'
					snake_case__	     =tokenizer.tokenize(_UpperCAmelCase						)
					self.assertListEqual(_UpperCAmelCase					,   ['こんにちは', '、', '世界', '。', 'こん', '##ばんは', '、', '世界', '。']						)
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

					snake_case__	     =os.path.join(self.tmpdirname					,   'tokenizer.bin'						)
					with open(_UpperCAmelCase					,   'wb'						) as handle:
							pickle.dump(_UpperCAmelCase					,   _UpperCAmelCase						)

					with open(_UpperCAmelCase					,   'rb'						) as handle:
							snake_case__	     =pickle.load(_UpperCAmelCase						)

					snake_case__	     =tokenizer_new.tokenize(_UpperCAmelCase						)

					self.assertListEqual(_UpperCAmelCase					,   _UpperCAmelCase						)






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(do_lower_case=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iphone', '8', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =JumanppTokenizer(normalize_text=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['ｱ', 'ｯ', 'ﾌ', 'ﾟ', 'ﾙ', 'ストア', 'で', 'iPhone', '８', '\u3000', 'が', '\u3000', '\u3000', '\u3000', '発売', 'さ', 'れた', '\u3000', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							int:
					snake_case__	     =JumanppTokenizer(trim_whitespace=_UpperCAmelCase						)

					self.assertListEqual(
					    tokenizer.tokenize(' \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  '						)					,   ['アップル', 'ストア', 'で', 'iPhone', '8', 'が', '発売', 'さ', 'れた', '。']					,   )






			@require_jumanpp
			def 						_lowercase  (     self						)							->							str:
					snake_case__	     =JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize('ありがとうございますm(_ _)ｍ見つけるのが大変です。'						)					,   ['ありがとう', 'ございます', 'm(_ _)m', '見つける', 'の', 'が', '大変です', '。']					,   )






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こんにちは', 'こん', 'にちは', 'ばんは', '##こん', '##にちは', '##ばんは']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =WordpieceTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こんにちは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは'						)					,   ['こん', '##ばんは']						)

					self.assertListEqual(tokenizer.tokenize('こんばんは こんばんにちは こんにちは'						)					,   ['こん', '##ばんは', '[UNK]', 'こんにちは']						)






			def 						_lowercase  (     self						)							->							Tuple:
					snake_case__	     =BertJapaneseTokenizer.from_pretrained('nlp-waseda/roberta-base-japanese-with-auto-jumanpp'						)
					snake_case__	     =tokenizer.subword_tokenizer

					snake_case__	     =subword_tokenizer.tokenize('国境 の 長い トンネル を 抜ける と 雪国 であった 。'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁国境', '▁の', '▁長い', '▁トンネル', '▁を', '▁抜ける', '▁と', '▁雪', '国', '▁であった', '▁。']						)

					snake_case__	     =subword_tokenizer.tokenize('こんばんは こんばん にち は こんにちは'						)
					self.assertListEqual(_UpperCAmelCase					,   ['▁こん', 'ばん', 'は', '▁こん', 'ばん', '▁に', 'ち', '▁は', '▁こんにちは']						)






			def 						_lowercase  (     self						)							->							Optional[int]:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( snake_case__   ,      unittest.TestCase     ):
			a_	: int									=      BertJapaneseTokenizer
			a_	: Optional[Any]									=      False






			def 						_lowercase  (     self						)							->							Any:
					super().setUp()

					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     =os.path.join(self.tmpdirname					,   VOCAB_FILES_NAMES['vocab_file']						)
					with open(self.vocab_file					,   'w'					,   encoding='utf-8'						) as vocab_writer:
							vocab_writer.write(''.join([x + '\n' for x in vocab_tokens]						)						)






			def 						_lowercase  (     self					,   **_UpperCAmelCase						)							->							Union[str, Any]:
					return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,   subword_tokenizer_type='character'					,   **_UpperCAmelCase						)






			def 						_lowercase  (     self					,   _UpperCAmelCase						)							->							Optional[Any]:
					snake_case__	     ='こんにちは、世界。 \nこんばんは、世界。'
					snake_case__	     ='こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。'
					return input_text, output_text






			def 						_lowercase  (     self						)							->							List[Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Union[str, Any]:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Tuple:
					pass  # TODO add if relevant






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class(self.vocab_file					,   subword_tokenizer_type='character'						)

					snake_case__	     =tokenizer.tokenize('こんにちは、世界。 \nこんばんは、世界。'						)
					self.assertListEqual(
					    _UpperCAmelCase					,   ['こ', 'ん', 'に', 'ち', 'は', '、', '世', '界', '。', 'こ', 'ん', 'ば', 'ん', 'は', '、', '世', '界', '。']						)
					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_UpperCAmelCase						)					,   [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     =['[UNK]', '[CLS]', '[SEP]', 'こ', 'ん', 'に', 'ち', 'は', 'ば', '世', '界', '、', '。']

					snake_case__	     ={}
					for i, token in enumerate(_UpperCAmelCase						):
							snake_case__	     =i
					snake_case__	     =CharacterTokenizer(vocab=_UpperCAmelCase					,   unk_token='[UNK]'						)

					self.assertListEqual(tokenizer.tokenize(''						)					,   []						)

					self.assertListEqual(tokenizer.tokenize('こんにちは'						)					,   ['こ', 'ん', 'に', 'ち', 'は']						)

					self.assertListEqual(tokenizer.tokenize('こんにちほ'						)					,   ['こ', 'ん', 'に', 'ち', '[UNK]']						)






			def 						_lowercase  (     self						)							->							Any:
					snake_case__	     =self.tokenizer_class.from_pretrained('cl-tohoku/bert-base-japanese-char'						)

					snake_case__	     =tokenizer.encode('ありがとう。'					,   add_special_tokens=_UpperCAmelCase						)
					snake_case__	     =tokenizer.encode('どういたしまして。'					,   add_special_tokens=_UpperCAmelCase						)

					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase						)
					snake_case__	     =tokenizer.build_inputs_with_special_tokens(_UpperCAmelCase					,   _UpperCAmelCase						)

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							Optional[Any]:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					snake_case__	     =AutoTokenizer.from_pretrained(_UpperCAmelCase						)
					self.assertIsInstance(_UpperCAmelCase					,   _UpperCAmelCase						)





class 							a__( unittest.TestCase     ):






			def 						_lowercase  (     self						)							->							int:
					snake_case__	     ='cl-tohoku/bert-base-japanese'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)
					snake_case__	     ='bert-base-cased'
					with self.assertLogs('transformers'					,   level='WARNING'						) as cm:
							BertJapaneseTokenizer.from_pretrained(_UpperCAmelCase						)
							self.assertTrue(
							    cm.records[0].message.startswith(
							        'The tokenizer class you load from this checkpoint is not the same type as the class this function'
							        ' is called from.'						)						)




