



import argparse
import os

import gluonnlp as nlp
import mxnet as mx
import numpy as np
import torch
from gluonnlp.base import get_home_dir
from gluonnlp.model.bert import BERTEncoder
from gluonnlp.model.utils import _load_vocab
from gluonnlp.vocab import Vocab
from packaging import version
from torch import nn

from transformers import BertConfig, BertForMaskedLM, BertModel, RobertaTokenizer
from transformers.models.bert.modeling_bert import (
    BertIntermediate,
    BertLayer,
    BertOutput,
    BertSelfAttention,
    BertSelfOutput,
)
from transformers.utils import logging


if version.parse(nlp.__version__) != version.parse("""0.8.3"""):
					raise Exception("""requires gluonnlp == 0.8.3""")

if version.parse(mx.__version__) != version.parse("""1.5.0"""):
					raise Exception("""requires mxnet == 1.5.0""")

logging.set_verbosity_info()
SCREAMING_SNAKE_CASE_				= logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_				= """The Nymphenburg Palace is a beautiful palace in Munich!"""



def        lowerCAmelCase_						(							SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	)			->      Any:
		a_     :      List[str] 						=     {
		    "attention_cell": "multi_head",
		    "num_layers": 4,
		    "units": 1_024,
		    "hidden_size": 768,
		    "max_length": 512,
		    "num_heads": 8,
		    "scaled": True,
		    "dropout": 0.1,
		    "use_residual": True,
		    "embed_size": 1_024,
		    "embed_dropout": 0.1,
		    "word_embed": None,
		    "layer_norm_eps": 1e-5,
		    "token_type_vocab_size": 2,
		}

		a_     :      Optional[int] 						=     bort_4_8_768_1024_hparams

		# Let's construct the original Bort model here
		# Taken from official BERT implementation, see:
		# https://github.com/alexa/bort/blob/master/bort/bort.py
		a_     :      int 						=     BERTEncoder(
		    attention_cell=predefined_args["attention_cell"], num_layers=predefined_args["num_layers"], units=predefined_args["units"], hidden_size=predefined_args["hidden_size"], max_length=predefined_args["max_length"], num_heads=predefined_args["num_heads"], scaled=predefined_args["scaled"], dropout=predefined_args["dropout"], output_attention=SCREAMING_SNAKE_CASE__, output_all_encodings=SCREAMING_SNAKE_CASE__, use_residual=predefined_args["use_residual"], activation=predefined_args.get("activation", "gelu"	), layer_norm_eps=predefined_args.get("layer_norm_eps", SCREAMING_SNAKE_CASE__	), )

		# Vocab information needs to be fetched first
		# It's the same as RoBERTa, so RobertaTokenizer can be used later
		a_     :      str 						=     "openwebtext_ccnews_stories_books_cased"

		# Specify download folder to Gluonnlp's vocab
		a_     :      Tuple 						=     os.path.join(get_home_dir(), "models"	)
		a_     :      str 						=     _load_vocab(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, cls=SCREAMING_SNAKE_CASE__	)

		a_     :      Tuple 						=     nlp.model.BERTModel(
		    SCREAMING_SNAKE_CASE__, len(SCREAMING_SNAKE_CASE__	), units=predefined_args["units"], embed_size=predefined_args["embed_size"], embed_dropout=predefined_args["embed_dropout"], word_embed=predefined_args["word_embed"], use_pooler=SCREAMING_SNAKE_CASE__, use_token_type_embed=SCREAMING_SNAKE_CASE__, token_type_vocab_size=predefined_args["token_type_vocab_size"], use_classifier=SCREAMING_SNAKE_CASE__, use_decoder=SCREAMING_SNAKE_CASE__, )

		original_bort.load_parameters(SCREAMING_SNAKE_CASE__, cast_dtype=SCREAMING_SNAKE_CASE__, ignore_extra=SCREAMING_SNAKE_CASE__	)
		a_     :      Optional[Any] 						=     original_bort._collect_params_with_prefix()

		# Build our config 🤗
		a_     :      str 						=     {
		    "architectures": ["BertForMaskedLM"],
		    "attention_probs_dropout_prob": predefined_args["dropout"],
		    "hidden_act": "gelu",
		    "hidden_dropout_prob": predefined_args["dropout"],
		    "hidden_size": predefined_args["embed_size"],
		    "initializer_range": 0.02,
		    "intermediate_size": predefined_args["hidden_size"],
		    "layer_norm_eps": predefined_args["layer_norm_eps"],
		    "max_position_embeddings": predefined_args["max_length"],
		    "model_type": "bort",
		    "num_attention_heads": predefined_args["num_heads"],
		    "num_hidden_layers": predefined_args["num_layers"],
		    "pad_token_id": 1,  # 2 = BERT, 1 = RoBERTa
		    "type_vocab_size": 1,  # 2 = BERT, 1 = RoBERTa
		    "vocab_size": len(SCREAMING_SNAKE_CASE__	),
		}

		a_     :      Any 						=     BertConfig.from_dict(SCREAMING_SNAKE_CASE__	)
		a_     :      Dict 						=     BertForMaskedLM(SCREAMING_SNAKE_CASE__	)
		hf_bort_model.eval()

		# Parameter mapping table (Gluonnlp to Transformers)
		# * denotes layer index
		#
		# | Gluon Parameter                                                | Transformers Parameter
		# | -------------------------------------------------------------- | ----------------------
		# | `encoder.layer_norm.beta`                                      | `bert.embeddings.LayerNorm.bias`
		# | `encoder.layer_norm.gamma`                                     | `bert.embeddings.LayerNorm.weight`
		# | `encoder.position_weight`                                      | `bert.embeddings.position_embeddings.weight`
		# | `word_embed.0.weight`                                          | `bert.embeddings.word_embeddings.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_key.bias`     | `bert.encoder.layer.*.attention.self.key.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_key.weight`   | `bert.encoder.layer.*.attention.self.key.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_query.bias`   | `bert.encoder.layer.*.attention.self.query.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_query.weight` | `bert.encoder.layer.*.attention.self.query.weight`
		# | `encoder.transformer_cells.*.attention_cell.proj_value.bias`   | `bert.encoder.layer.*.attention.self.value.bias`
		# | `encoder.transformer_cells.*.attention_cell.proj_value.weight` | `bert.encoder.layer.*.attention.self.value.weight`
		# | `encoder.transformer_cells.*.ffn.ffn_2.bias`                   | `bert.encoder.layer.*.attention.output.dense.bias`
		# | `encoder.transformer_cells.*.ffn.ffn_2.weight`                 | `bert.encoder.layer.*.attention.output.dense.weight`
		# | `encoder.transformer_cells.*.layer_norm.beta`                  | `bert.encoder.layer.*.attention.output.LayerNorm.bias`
		# | `encoder.transformer_cells.*.layer_norm.gamma`                 | `bert.encoder.layer.*.attention.output.LayerNorm.weight`
		# | `encoder.transformer_cells.*.ffn.ffn_1.bias`                   | `bert.encoder.layer.*.intermediate.dense.bias`
		# | `encoder.transformer_cells.*.ffn.ffn_1.weight`                 | `bert.encoder.layer.*.intermediate.dense.weight`
		# | `encoder.transformer_cells.*.ffn.layer_norm.beta`              | `bert.encoder.layer.*.output.LayerNorm.bias`
		# | `encoder.transformer_cells.*.ffn.layer_norm.gamma`             | `bert.encoder.layer.*.output.LayerNorm.weight`
		# | `encoder.transformer_cells.*.proj.bias`                        | `bert.encoder.layer.*.output.dense.bias`
		# | `encoder.transformer_cells.*.proj.weight`                      | `bert.encoder.layer.*.output.dense.weight`

		# Helper function to convert MXNET Arrays to PyTorch
		def to_torch(SCREAMING_SNAKE_CASE__	) -> nn.Parameter:
				return nn.Parameter(torch.FloatTensor(mx_array.data().asnumpy()	)	)

		# Check param shapes and map new HF param back
		def check_and_map_params(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__	):
				a_     :      Optional[Any] 						=     hf_param.shape

				a_     :      str 						=     to_torch(params[gluon_param]	)
				a_     :      Union[str, Any] 						=     gluon_param.shape

				assert (
				    shape_hf == shape_gluon
				), F"""The gluon parameter {gluon_param} has shape {shape_gluon}, but expects shape {shape_hf} for Transformers"""

				return gluon_param

		a_     :      Optional[Any] 						=     check_and_map_params(
		    hf_bort_model.bert.embeddings.word_embeddings.weight, "word_embed.0.weight"	)
		a_     :      Optional[Any] 						=     check_and_map_params(
		    hf_bort_model.bert.embeddings.position_embeddings.weight, "encoder.position_weight"	)
		a_     :      List[str] 						=     check_and_map_params(
		    hf_bort_model.bert.embeddings.LayerNorm.bias, "encoder.layer_norm.beta"	)
		a_     :      Union[str, Any] 						=     check_and_map_params(
		    hf_bort_model.bert.embeddings.LayerNorm.weight, "encoder.layer_norm.gamma"	)

		# Inspired by RoBERTa conversion script, we just zero them out (Bort does not use them)
		a_     :      Dict 						=     torch.zeros_like(
		    hf_bort_model.bert.embeddings.token_type_embeddings.weight.data	)

		for i in range(hf_bort_config.num_hidden_layers	):
				a_     :      BertLayer 						=     hf_bort_model.bert.encoder.layer[i]

				# self attention
				a_     :      BertSelfAttention 						=     layer.attention.self

				a_     :      int 						=     check_and_map_params(
				    self_attn.key.bias.data, F"""encoder.transformer_cells.{i}.attention_cell.proj_key.bias"""	)

				a_     :      Dict 						=     check_and_map_params(
				    self_attn.key.weight.data, F"""encoder.transformer_cells.{i}.attention_cell.proj_key.weight"""	)
				a_     :      Dict 						=     check_and_map_params(
				    self_attn.query.bias.data, F"""encoder.transformer_cells.{i}.attention_cell.proj_query.bias"""	)
				a_     :      str 						=     check_and_map_params(
				    self_attn.query.weight.data, F"""encoder.transformer_cells.{i}.attention_cell.proj_query.weight"""	)
				a_     :      Optional[int] 						=     check_and_map_params(
				    self_attn.value.bias.data, F"""encoder.transformer_cells.{i}.attention_cell.proj_value.bias"""	)
				a_     :      List[str] 						=     check_and_map_params(
				    self_attn.value.weight.data, F"""encoder.transformer_cells.{i}.attention_cell.proj_value.weight"""	)

				# self attention output
				a_     :      BertSelfOutput 						=     layer.attention.output

				a_     :      Any 						=     check_and_map_params(
				    self_output.dense.bias, F"""encoder.transformer_cells.{i}.proj.bias"""	)
				a_     :      Dict 						=     check_and_map_params(
				    self_output.dense.weight, F"""encoder.transformer_cells.{i}.proj.weight"""	)
				a_     :      int 						=     check_and_map_params(
				    self_output.LayerNorm.bias, F"""encoder.transformer_cells.{i}.layer_norm.beta"""	)
				a_     :      Any 						=     check_and_map_params(
				    self_output.LayerNorm.weight, F"""encoder.transformer_cells.{i}.layer_norm.gamma"""	)

				# intermediate
				a_     :      BertIntermediate 						=     layer.intermediate

				a_     :      int 						=     check_and_map_params(
				    intermediate.dense.bias, F"""encoder.transformer_cells.{i}.ffn.ffn_1.bias"""	)
				a_     :      Union[str, Any] 						=     check_and_map_params(
				    intermediate.dense.weight, F"""encoder.transformer_cells.{i}.ffn.ffn_1.weight"""	)

				# output
				a_     :      BertOutput 						=     layer.output

				a_     :      int 						=     check_and_map_params(
				    bert_output.dense.bias, F"""encoder.transformer_cells.{i}.ffn.ffn_2.bias"""	)
				a_     :      Dict 						=     check_and_map_params(
				    bert_output.dense.weight, F"""encoder.transformer_cells.{i}.ffn.ffn_2.weight"""	)
				a_     :      Any 						=     check_and_map_params(
				    bert_output.LayerNorm.bias, F"""encoder.transformer_cells.{i}.ffn.layer_norm.beta"""	)
				a_     :      Union[str, Any] 						=     check_and_map_params(
				    bert_output.LayerNorm.weight, F"""encoder.transformer_cells.{i}.ffn.layer_norm.gamma"""	)

		# Save space and energy 🎄
		hf_bort_model.half()

		# Compare output of both models
		a_     :      Optional[Any] 						=     RobertaTokenizer.from_pretrained("roberta-base"	)

		a_     :      Any 						=     tokenizer.encode_plus(SCREAMING_SNAKE_CASE__	)["input_ids"]

		# Get gluon output
		a_     :      Union[str, Any] 						=     mx.nd.array([input_ids]	)
		a_     :      Optional[int] 						=     original_bort(inputs=SCREAMING_SNAKE_CASE__, token_types=[]	)

		# Get Transformer output (save and reload model again)
		hf_bort_model.save_pretrained(SCREAMING_SNAKE_CASE__	)
		a_     :      str 						=     BertModel.from_pretrained(SCREAMING_SNAKE_CASE__	)
		hf_bort_model.eval()

		a_     :      str 						=     tokenizer.encode_plus(SCREAMING_SNAKE_CASE__, return_tensors="pt"	)
		a_     :      Tuple 						=     hf_bort_model(**SCREAMING_SNAKE_CASE__	)[0]

		a_     :      Any 						=     output_gluon[0].asnumpy()
		a_     :      Tuple 						=     output_hf[0].detach().numpy()

		a_     :      Optional[int] 						=     np.max(np.abs(hf_layer - gluon_layer	)	).item()
		a_     :      List[Any] 						=     np.allclose(SCREAMING_SNAKE_CASE__, SCREAMING_SNAKE_CASE__, atol=1e-3	)

		if success:
				print("✔️ Both model do output the same tensors"	)
		else:
				print("❌ Both model do **NOT** output the same tensors"	)
				print("Absolute difference is:", SCREAMING_SNAKE_CASE__	)


if __name__ == "__main__":
					SCREAMING_SNAKE_CASE_				= argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    """--bort_checkpoint_path""", default=None, type=str, required=True, help="""Path the official Bort params file."""
					)
					parser.add_argument(
					    """--pytorch_dump_folder_path""", default=None, type=str, required=True, help="""Path to the output PyTorch model."""
					)
					SCREAMING_SNAKE_CASE_				= parser.parse_args()
					convert_bort_checkpoint_to_pytorch(args.bort_checkpoint_path, args.pytorch_dump_folder_path)




"""simple docstring"""
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					BertJapaneseTokenizer
							__lowerCAmelCase    					=					False
							__lowerCAmelCase    					=					True






							def snake_case_  (   self      ):
									super().setUp()

									a_     :      str 						=     [
									    "[UNK]",
									    "[CLS]",
									    "[SEP]",
									    "こんにちは",
									    "こん",
									    "にちは",
									    "ばんは",
									    "##こん",
									    "##にちは",
									    "##ばんは",
									    "世界",
									    "##世界",
									    "、",
									    "##、",
									    "。",
									    "##。",
									]

									a_     :      Tuple 						=     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]      )
									with open(self.vocab_file			,     "w"			,     encoding="utf-8"      ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )






							def snake_case_  (   self			,     a_      ):
									a_     :      Dict 						=     "こんにちは、世界。 \nこんばんは、世界。"
									a_     :      int 						=     "こんにちは 、 世界 。 こんばんは 、 世界 。"
									return input_text, output_text






							def snake_case_  (   self			,     a_      ):
									a_						,				a_     :      List[str] 						=     self.get_input_output_texts(a_      )
									a_     :      Tuple 						=     tokenizer.encode(a_			,     add_special_tokens=a_      )
									a_     :      Dict 						=     tokenizer.decode(a_			,     clean_up_tokenization_spaces=a_      )
									return text, ids






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									a_     :      int 						=     self.tokenizer_class(self.vocab_file      )

									a_     :      List[Any] 						=     tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="mecab"      )
									self.assertIsNotNone(a_      )

									a_     :      Union[str, Any] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      Union[str, Any] 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      Any 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Optional[Any] 						=     pickle.load(a_      )

									a_     :      int 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							def snake_case_  (   self      ):
									a_     :      Tuple 						=     MecabTokenizer(mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      Dict 						=     MecabTokenizer(mecab_dic="unidic_lite"      )
									except ModuleNotFoundError:
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      List[Any] 						=     MecabTokenizer(mecab_dic="unidic"      )
									except ModuleNotFoundError:
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     MecabTokenizer(do_lower_case=a_			,     mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							def snake_case_  (   self      ):
									try:
											a_     :      int 						=     MecabTokenizer(
											    do_lower_case=a_			,     normalize_text=a_			,     mecab_option="-d /usr/local/lib/mecab/dic/jumandic"      )
									except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     MecabTokenizer(normalize_text=a_			,     mecab_dic="ipadic"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="sudachi"      )
									self.assertIsNotNone(a_      )

									a_     :      Optional[Any] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      str 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      Dict 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Optional[Any] 						=     pickle.load(a_      )

									a_     :      int 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     SudachiTokenizer(sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Any 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="A"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国", "人", "参政", "権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="B"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国人", "参政権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      int 						=     SudachiTokenizer(sudachi_dict_type="core"			,     sudachi_split_mode="C"      )

									self.assertListEqual(tokenizer.tokenize("外国人参政権"      )			,     ["外国人参政権"]      )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      List[str] 						=     SudachiTokenizer(do_lower_case=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      str 						=     SudachiTokenizer(normalize_text=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     [" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]			,     )






							@require_sudachi
							def snake_case_  (   self      ):
									a_     :      Optional[int] 						=     SudachiTokenizer(trim_whitespace=a_			,     sudachi_dict_type="core"      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     self.tokenizer_class(self.vocab_file			,     word_tokenizer_type="jumanpp"      )
									self.assertIsNotNone(a_      )

									a_     :      Optional[int] 						=     "こんにちは、世界。\nこんばんは、世界。"
									a_     :      Any 						=     tokenizer.tokenize(a_      )
									self.assertListEqual(a_			,     ["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]      )
									self.assertListEqual(tokenizer.convert_tokens_to_ids(a_      )			,     [3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]      )

									a_     :      int 						=     os.path.join(self.tmpdirname			,     "tokenizer.bin"      )
									with open(a_			,     "wb"      ) as handle:
											pickle.dump(a_			,     a_      )

									with open(a_			,     "rb"      ) as handle:
											a_     :      Tuple 						=     pickle.load(a_      )

									a_     :      Union[str, Any] 						=     tokenizer_new.tokenize(a_      )

									self.assertListEqual(a_			,     a_      )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Any 						=     JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     JumanppTokenizer(do_lower_case=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      int 						=     JumanppTokenizer(normalize_text=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Optional[Any] 						=     JumanppTokenizer(trim_whitespace=a_      )

									self.assertListEqual(
									    tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "      )			,     ["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]			,     )






							@require_jumanpp
							def snake_case_  (   self      ):
									a_     :      Any 						=     JumanppTokenizer()

									self.assertListEqual(
									    tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"      )			,     ["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]			,     )






							def snake_case_  (   self      ):
									a_     :      str 						=     ["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

									a_     :      int 						=     {}
									for i, token in enumerate(a_      ):
											a_     :      Optional[int] 						=     i
									a_     :      Optional[int] 						=     WordpieceTokenizer(vocab=a_			,     unk_token="[UNK]"      )

									self.assertListEqual(tokenizer.tokenize(""      )			,     []      )

									self.assertListEqual(tokenizer.tokenize("こんにちは"      )			,     ["こんにちは"]      )

									self.assertListEqual(tokenizer.tokenize("こんばんは"      )			,     ["こん", "##ばんは"]      )

									self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"      )			,     ["こん", "##ばんは", "[UNK]", "こんにちは"]      )






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"      )
									a_     :      Any 						=     tokenizer.subword_tokenizer

									a_     :      Union[str, Any] 						=     subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"      )
									self.assertListEqual(a_			,     ["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]      )

									a_     :      int 						=     subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"      )
									self.assertListEqual(a_			,     ["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]      )






							def snake_case_  (   self      ):
									a_     :      List[str] 						=     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"      )

									a_     :      Any 						=     tokenizer.encode("ありがとう。"			,     add_special_tokens=a_      )
									a_     :      Optional[Any] 						=     tokenizer.encode("どういたしまして。"			,     add_special_tokens=a_      )

									a_     :      Union[str, Any] 						=     tokenizer.build_inputs_with_special_tokens(a_      )
									a_     :      Tuple 						=     tokenizer.build_inputs_with_special_tokens(a_			,     a_      )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			snake_case_						(						a_     ,unittest.TestCase				):
							__lowerCAmelCase    					=					BertJapaneseTokenizer
							__lowerCAmelCase    					=					False






							def snake_case_  (   self      ):
									super().setUp()

									a_     :      Any 						=     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									a_     :      str 						=     os.path.join(self.tmpdirname			,     VOCAB_FILES_NAMES["vocab_file"]      )
									with open(self.vocab_file			,     "w"			,     encoding="utf-8"      ) as vocab_writer:
											vocab_writer.write("".join([x + "\n" for x in vocab_tokens]      )      )






							def snake_case_  (   self			,     **a_      ):
									return BertJapaneseTokenizer.from_pretrained(self.tmpdirname			,     subword_tokenizer_type="character"			,     **a_      )






							def snake_case_  (   self			,     a_      ):
									a_     :      int 						=     "こんにちは、世界。 \nこんばんは、世界。"
									a_     :      List[str] 						=     "こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
									return input_text, output_text






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									pass  # TODO add if relevant






							def snake_case_  (   self      ):
									a_     :      Dict 						=     self.tokenizer_class(self.vocab_file			,     subword_tokenizer_type="character"      )

									a_     :      int 						=     tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"      )
									self.assertListEqual(
									    a_			,     ["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]      )
									self.assertListEqual(
									    tokenizer.convert_tokens_to_ids(a_      )			,     [3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]      )






							def snake_case_  (   self      ):
									a_     :      Union[str, Any] 						=     ["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

									a_     :      Any 						=     {}
									for i, token in enumerate(a_      ):
											a_     :      str 						=     i
									a_     :      List[str] 						=     CharacterTokenizer(vocab=a_			,     unk_token="[UNK]"      )

									self.assertListEqual(tokenizer.tokenize(""      )			,     []      )

									self.assertListEqual(tokenizer.tokenize("こんにちは"      )			,     ["こ", "ん", "に", "ち", "は"]      )

									self.assertListEqual(tokenizer.tokenize("こんにちほ"      )			,     ["こ", "ん", "に", "ち", "[UNK]"]      )






							def snake_case_  (   self      ):
									a_     :      str 						=     self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"      )

									a_     :      Optional[int] 						=     tokenizer.encode("ありがとう。"			,     add_special_tokens=a_      )
									a_     :      Tuple 						=     tokenizer.encode("どういたしまして。"			,     add_special_tokens=a_      )

									a_     :      Optional[Any] 						=     tokenizer.build_inputs_with_special_tokens(a_      )
									a_     :      Any 						=     tokenizer.build_inputs_with_special_tokens(a_			,     a_      )

									# 2 is for "[CLS]", 3 is for "[SEP]"
									assert encoded_sentence == [2] + text + [3]
									assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self      ):
									a_     :      List[Any] 						=     "cl-tohoku/bert-base-japanese"
									a_     :      Dict 						=     AutoTokenizer.from_pretrained(a_      )
									self.assertIsInstance(a_			,     a_      )




class 			snake_case_						(						unittest.TestCase				):






							def snake_case_  (   self      ):
									a_     :      Any 						=     "cl-tohoku/bert-base-japanese"
									with self.assertLogs("transformers"			,     level="WARNING"      ) as cm:
											BertTokenizer.from_pretrained(a_      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
											        " is called from."      )      )
									a_     :      Any 						=     "bert-base-cased"
									with self.assertLogs("transformers"			,     level="WARNING"      ) as cm:
											BertJapaneseTokenizer.from_pretrained(a_      )
											self.assertTrue(
											    cm.records[0].message.startswith(
											        "The tokenizer class you load from this checkpoint is not the same type as the class this function"
											        " is called from."      )      )