





'''simple docstring'''




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin




@custom_tokenizers
class      UpperCamelCase__					(					lowercase_						,					unittest.TestCase			):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__				=	BertJapaneseTokenizer
					SCREAMING_SNAKE_CASE__				=	False
					SCREAMING_SNAKE_CASE__				=	True


					def lowerCamelCase_			(						self				:			List[Any]						):

							'''simple docstring'''



							super().setUp()

							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			[
							    "[UNK]",
							    "[CLS]",
							    "[SEP]",
							    "こんにちは",
							    "こん",
							    "にちは",
							    "ばんは",
							    "##こん",
							    "##にちは",
							    "##ばんは",
							    "世界",
							    "##世界",
							    "、",
							    "##、",
							    "。",
							    "##。",
							]

							SCREAMING_SNAKE_CASE :				str		=			os.path.join(self.tmpdirname      ,						VOCAB_FILES_NAMES["""vocab_file"""]						)
							with open(self.vocab_file      ,						"""w"""      ,						encoding="""utf-8"""						) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)


					def lowerCamelCase_			(						self				:			Optional[Any]      ,						lowerCamelCase_				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			"こんにちは、世界。 \nこんばんは、世界。"
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			"こんにちは 、 世界 。 こんばんは 、 世界 。"
							return input_text, output_text


					def lowerCamelCase_			(						self				:			Optional[Any]      ,						lowerCamelCase_				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				int		=			self.get_input_output_texts(snake_case__						)
							SCREAMING_SNAKE_CASE :				int		=			tokenizer.encode(snake_case__      ,						add_special_tokens=snake_case__						)
							SCREAMING_SNAKE_CASE :				str		=			tokenizer.decode(snake_case__      ,						clean_up_tokenization_spaces=snake_case__						)
							return text, ids


					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							pass  # TODO add if relevant


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							pass  # TODO add if relevant


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							pass  # TODO add if relevant


					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			self.tokenizer_class(self.vocab_file						)

							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""						)
							self.assertListEqual(snake_case__      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__						)      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)


					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Any		=			self.tokenizer_class(self.vocab_file      ,						word_tokenizer_type="""mecab"""						)
							self.assertIsNotNone(snake_case__						)

							SCREAMING_SNAKE_CASE :				Optional[Any]		=			"こんにちは、世界。\nこんばんは、世界。"
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			tokenizer.tokenize(snake_case__						)
							self.assertListEqual(snake_case__      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__						)      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

							SCREAMING_SNAKE_CASE :				List[Any]		=			os.path.join(self.tmpdirname      ,						"""tokenizer.bin"""						)
							with open(snake_case__      ,						"""wb"""						) as handle:
									pickle.dump(snake_case__      ,						snake_case__						)

							with open(snake_case__      ,						"""rb"""						) as handle:
									SCREAMING_SNAKE_CASE :				Tuple		=			pickle.load(snake_case__						)

							SCREAMING_SNAKE_CASE :				int		=			tokenizer_new.tokenize(snake_case__						)

							self.assertListEqual(snake_case__      ,						snake_case__						)


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Dict		=			MecabTokenizer(mecab_dic="""ipadic"""						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)


					def lowerCamelCase_			(						self				:			Any						):

							'''simple docstring'''



							try:
									SCREAMING_SNAKE_CASE :				List[Any]		=			MecabTokenizer(mecab_dic="""unidic_lite"""						)
							except ModuleNotFoundError:
									return

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)


					def lowerCamelCase_			(						self				:			List[str]						):

							'''simple docstring'''



							try:
									SCREAMING_SNAKE_CASE :				Tuple		=			MecabTokenizer(mecab_dic="""unidic"""						)
							except ModuleNotFoundError:
									return

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)


					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[Any]		=			MecabTokenizer(do_lower_case=snake_case__      ,						mecab_dic="""ipadic"""						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)


					def lowerCamelCase_			(						self				:			Any						):

							'''simple docstring'''



							try:
									SCREAMING_SNAKE_CASE :				List[str]		=			MecabTokenizer(
									    do_lower_case=snake_case__      ,						normalize_text=snake_case__      ,						mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""						)
							except RuntimeError:
									# if dict doesn't exist in the system, previous code raises this error.
									return

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				str		=			MecabTokenizer(normalize_text=snake_case__      ,						mecab_dic="""ipadic"""						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]      ,						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				int		=			self.tokenizer_class(self.vocab_file      ,						word_tokenizer_type="""sudachi"""						)
							self.assertIsNotNone(snake_case__						)

							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			"こんにちは、世界。\nこんばんは、世界。"
							SCREAMING_SNAKE_CASE :				Dict		=			tokenizer.tokenize(snake_case__						)
							self.assertListEqual(snake_case__      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__						)      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

							SCREAMING_SNAKE_CASE :				List[Any]		=			os.path.join(self.tmpdirname      ,						"""tokenizer.bin"""						)
							with open(snake_case__      ,						"""wb"""						) as handle:
									pickle.dump(snake_case__      ,						snake_case__						)

							with open(snake_case__      ,						"""rb"""						) as handle:
									SCREAMING_SNAKE_CASE :				Any		=			pickle.load(snake_case__						)

							SCREAMING_SNAKE_CASE :				Tuple		=			tokenizer_new.tokenize(snake_case__						)

							self.assertListEqual(snake_case__      ,						snake_case__						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			SudachiTokenizer(sudachi_dict_type="""core"""						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			Optional[int]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				int		=			SudachiTokenizer(sudachi_dict_type="""core"""      ,						sudachi_split_mode="""A"""						)

							self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)      ,						["""外国""", """人""", """参政""", """権"""]						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			SudachiTokenizer(sudachi_dict_type="""core"""      ,						sudachi_split_mode="""B"""						)

							self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)      ,						["""外国人""", """参政権"""]						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			SudachiTokenizer(sudachi_dict_type="""core"""      ,						sudachi_split_mode="""C"""						)

							self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)      ,						["""外国人参政権"""]						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			Optional[int]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[str]		=			SudachiTokenizer(do_lower_case=snake_case__      ,						sudachi_dict_type="""core"""						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]      ,						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[str]		=			SudachiTokenizer(normalize_text=snake_case__      ,						sudachi_dict_type="""core"""						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]      ,						)


					@require_sudachi
					def lowerCamelCase_			(						self				:			str						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			SudachiTokenizer(trim_whitespace=snake_case__      ,						sudachi_dict_type="""core"""						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]      ,						)


					@require_jumanpp
					def lowerCamelCase_			(						self				:			List[str]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.tokenizer_class(self.vocab_file      ,						word_tokenizer_type="""jumanpp"""						)
							self.assertIsNotNone(snake_case__						)

							SCREAMING_SNAKE_CASE :				str		=			"こんにちは、世界。\nこんばんは、世界。"
							SCREAMING_SNAKE_CASE :				Tuple		=			tokenizer.tokenize(snake_case__						)
							self.assertListEqual(snake_case__      ,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
							self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case__						)      ,						[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

							SCREAMING_SNAKE_CASE :				List[str]		=			os.path.join(self.tmpdirname      ,						"""tokenizer.bin"""						)
							with open(snake_case__      ,						"""wb"""						) as handle:
									pickle.dump(snake_case__      ,						snake_case__						)

							with open(snake_case__      ,						"""rb"""						) as handle:
									SCREAMING_SNAKE_CASE :				Optional[Any]		=			pickle.load(snake_case__						)

							SCREAMING_SNAKE_CASE :				List[str]		=			tokenizer_new.tokenize(snake_case__						)

							self.assertListEqual(snake_case__      ,						snake_case__						)


					@require_jumanpp
					def lowerCamelCase_			(						self				:			str						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[Any]		=			JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)


					@require_jumanpp
					def lowerCamelCase_			(						self				:			Any						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[str]		=			JumanppTokenizer(do_lower_case=snake_case__						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)


					@require_jumanpp
					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Tuple		=			JumanppTokenizer(normalize_text=snake_case__						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]      ,						)


					@require_jumanpp
					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			JumanppTokenizer(trim_whitespace=snake_case__						)

							self.assertListEqual(
							    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)      ,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]      ,						)


					@require_jumanpp
					def lowerCamelCase_			(						self				:			Optional[int]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Dict		=			JumanppTokenizer()

							self.assertListEqual(
							    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""						)      ,						["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]      ,						)


					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

							SCREAMING_SNAKE_CASE :				List[str]		=			{}
							for i, token in enumerate(snake_case__						):
									SCREAMING_SNAKE_CASE :				Optional[int]		=			i
							SCREAMING_SNAKE_CASE :				int		=			WordpieceTokenizer(vocab=snake_case__      ,						unk_token="""[UNK]"""						)

							self.assertListEqual(tokenizer.tokenize(""""""						)      ,						[]						)

							self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)      ,						["""こんにちは"""]						)

							self.assertListEqual(tokenizer.tokenize("""こんばんは"""						)      ,						["""こん""", """##ばんは"""]						)

							self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""						)      ,						["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]						)


					def lowerCamelCase_			(						self				:			List[str]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				str		=			BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""						)
							SCREAMING_SNAKE_CASE :				List[str]		=			tokenizer.subword_tokenizer

							SCREAMING_SNAKE_CASE :				Tuple		=			subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""						)
							self.assertListEqual(snake_case__      ,						["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]						)

							SCREAMING_SNAKE_CASE :				Any		=			subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""						)
							self.assertListEqual(snake_case__      ,						["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]						)



					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Any		=			self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""						)

							SCREAMING_SNAKE_CASE :				Tuple		=			tokenizer.encode("""ありがとう。"""      ,						add_special_tokens=snake_case__						)
							SCREAMING_SNAKE_CASE :				Dict		=			tokenizer.encode("""どういたしまして。"""      ,						add_special_tokens=snake_case__						)

							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			tokenizer.build_inputs_with_special_tokens(snake_case__						)
							SCREAMING_SNAKE_CASE :				List[str]		=			tokenizer.build_inputs_with_special_tokens(snake_case__      ,						snake_case__						)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      UpperCamelCase__					(					lowercase_						,					unittest.TestCase			):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__				=	BertJapaneseTokenizer
					SCREAMING_SNAKE_CASE__				=	False


					def lowerCamelCase_			(						self				:			Any						):

							'''simple docstring'''



							super().setUp()

							SCREAMING_SNAKE_CASE :				Optional[int]		=			["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

							SCREAMING_SNAKE_CASE :				Tuple		=			os.path.join(self.tmpdirname      ,						VOCAB_FILES_NAMES["""vocab_file"""]						)
							with open(self.vocab_file      ,						"""w"""      ,						encoding="""utf-8"""						) as vocab_writer:
									vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)


					def lowerCamelCase_			(						self				:			str      ,						**lowerCamelCase_				:			Dict						):

							'''simple docstring'''



							return BertJapaneseTokenizer.from_pretrained(self.tmpdirname      ,						subword_tokenizer_type="""character"""      ,						**snake_case__						)


					def lowerCamelCase_			(						self				:			Dict      ,						lowerCamelCase_				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Dict		=			"こんにちは、世界。 \nこんばんは、世界。"
							SCREAMING_SNAKE_CASE :				Tuple		=			"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
							return input_text, output_text


					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							pass  # TODO add if relevant


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							pass  # TODO add if relevant


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							pass  # TODO add if relevant


					def lowerCamelCase_			(						self				:			List[str]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				str		=			self.tokenizer_class(self.vocab_file      ,						subword_tokenizer_type="""character"""						)

							SCREAMING_SNAKE_CASE :				List[str]		=			tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""						)
							self.assertListEqual(
							    snake_case__      ,						["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]						)
							self.assertListEqual(
							    tokenizer.convert_tokens_to_ids(snake_case__						)      ,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)


					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[Any]		=			["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

							SCREAMING_SNAKE_CASE :				Any		=			{}
							for i, token in enumerate(snake_case__						):
									SCREAMING_SNAKE_CASE :				Union[str, Any]		=			i
							SCREAMING_SNAKE_CASE :				Any		=			CharacterTokenizer(vocab=snake_case__      ,						unk_token="""[UNK]"""						)

							self.assertListEqual(tokenizer.tokenize(""""""						)      ,						[]						)

							self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)      ,						["""こ""", """ん""", """に""", """ち""", """は"""]						)

							self.assertListEqual(tokenizer.tokenize("""こんにちほ"""						)      ,						["""こ""", """ん""", """に""", """ち""", """[UNK]"""]						)



					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[str]		=			self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""						)

							SCREAMING_SNAKE_CASE :				Tuple		=			tokenizer.encode("""ありがとう。"""      ,						add_special_tokens=snake_case__						)
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			tokenizer.encode("""どういたしまして。"""      ,						add_special_tokens=snake_case__						)

							SCREAMING_SNAKE_CASE :				Any		=			tokenizer.build_inputs_with_special_tokens(snake_case__						)
							SCREAMING_SNAKE_CASE :				int		=			tokenizer.build_inputs_with_special_tokens(snake_case__      ,						snake_case__						)

							# 2 is for "[CLS]", 3 is for "[SEP]"
							assert encoded_sentence == [2] + text + [3]
							assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class      UpperCamelCase__					(					unittest.TestCase			):




					"""simple docstring"""


					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			"cl-tohoku/bert-base-japanese"
							SCREAMING_SNAKE_CASE :				int		=			AutoTokenizer.from_pretrained(snake_case__						)
							self.assertIsInstance(snake_case__      ,						snake_case__						)







class      UpperCamelCase__					(					unittest.TestCase			):




					"""simple docstring"""


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			"cl-tohoku/bert-base-japanese"
							with self.assertLogs("""transformers"""      ,						level="""WARNING"""						) as cm:
									BertTokenizer.from_pretrained(snake_case__						)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""						)						)
							SCREAMING_SNAKE_CASE :				int		=			"bert-base-cased"
							with self.assertLogs("""transformers"""      ,						level="""WARNING"""						) as cm:
									BertJapaneseTokenizer.from_pretrained(snake_case__						)
									self.assertTrue(
									    cm.records[0].message.startswith(
									        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
									        """ is called from."""						)						)





'''simple docstring'''





def snake_case_     (					lowercase__ = "The quick brown fox jumps over the lazy dog"   ,       ):
  UpperCAmelCase__      :					Dict				       =							set()

  # Replace all the whitespace in our sentence
  UpperCAmelCase__      :					str				       =							input_str.replace(" "   ,       ""		)
  for alpha in input_str:
    if "a" <= alpha.lower() <= "z":
      frequency.add(alpha.lower()		)
  return len(lowercase__		) == 2_6







def snake_case_     (					lowercase__ = "The quick brown fox jumps over the lazy dog"   ,       ):
  UpperCAmelCase__      :					str				       =							[False] * 2_6
  for char in input_str:
    if char.islower():
      UpperCAmelCase__      :					List[Any]				       =							True
    elif char.isupper():
      UpperCAmelCase__      :					List[Any]				       =							True
  return all(lowercase__		)







def snake_case_     (					lowercase__ = "The quick brown fox jumps over the lazy dog"   ,       ):
  return len({char for char in input_str.lower() if char.isalpha()}		) == 2_6







def snake_case_     (					):
  from timeit import timeit

  UpperCAmelCase__      :					Union[str, Any]				       =							"from __main__ import is_pangram, is_pangram_faster, is_pangram_fastest"
  print(timeit("is_pangram()"   ,       setup=lowercase__		)		)
  print(timeit("is_pangram_faster()"   ,       setup=lowercase__		)		)
  print(timeit("is_pangram_fastest()"   ,       setup=lowercase__		)		)
  # 5.348480500048026, 2.6477354579837993, 1.8470395830227062
  # 5.036091582966037, 2.644472333951853,  1.8869528750656173


if __name__ == "__main__":
      import doctest

      doctest.testmod()
      benchmark()



