



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class 		snake_case						(	SCREAMING_SNAKE_CASE_					,unittest.TestCase  ):



   '''simple docstring'''






   snake_case_					: Optional[Any]          =		BertJapaneseTokenizer
   snake_case_					: Optional[int]          =		False
   snake_case_					: Optional[int]          =		True

   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    super().setUp()

    _snake_case    :  List[str]					=					[
        """[UNK]""",
        """[CLS]""",
        """[SEP]""",
        """こんにちは""",
        """こん""",
        """にちは""",
        """ばんは""",
        """##こん""",
        """##にちは""",
        """##ばんは""",
        """世界""",
        """##世界""",
        """、""",
        """##、""",
        """。""",
        """##。""",
    ]

    _snake_case    :  Dict					=					os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["""vocab_file"""])
    with open(self.vocab_file				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))

   def      UpperCamelCase_  (					self							:			Union[str, Any]				,		lowerCAmelCase							:			str)				->				int:

    """simple docstring"""







    _snake_case    :  List[Any]					=					"""こんにちは、世界。 \nこんばんは、世界。"""
    _snake_case    :  Optional[Any]					=					"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
    return input_text, output_text

   def      UpperCamelCase_  (					self							:			Tuple				,		lowerCAmelCase							:			Any)				->				Optional[Any]:

    """simple docstring"""







    _snake_case	,    _snake_case    :  Dict					=					self.get_input_output_texts(lowerCAmelCase)
    _snake_case    :  Optional[Any]					=					tokenizer.encode(lowerCAmelCase				,		add_special_tokens=lowerCAmelCase)
    _snake_case    :  Tuple					=					tokenizer.decode(lowerCAmelCase				,		clean_up_tokenization_spaces=lowerCAmelCase)
    return text, ids

   def      UpperCamelCase_  (					self							:			List[Any])				->				int:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			List[Any])				->				Optional[Any]:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			List[Any])				->				Dict:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			Dict)				->				Any:

    """simple docstring"""







    _snake_case    :  List[str]					=					self.tokenizer_class(self.vocab_file)

    _snake_case    :  List[Any]					=					tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。""")
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

   def      UpperCamelCase_  (					self							:			int)				->				Dict:

    """simple docstring"""







    _snake_case    :  Tuple					=					self.tokenizer_class(self.vocab_file				,		word_tokenizer_type="""mecab""")
    self.assertIsNotNone(lowerCAmelCase)

    _snake_case    :  List[str]					=					"""こんにちは、世界。\nこんばんは、世界。"""
    _snake_case    :  Dict					=					tokenizer.tokenize(lowerCAmelCase)
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

    _snake_case    :  Tuple					=					os.path.join(self.tmpdirname				,		"""tokenizer.bin""")
    with open(lowerCAmelCase				,		"""wb""") as handle:
     pickle.dump(lowerCAmelCase				,		lowerCAmelCase)

    with open(lowerCAmelCase				,		"""rb""") as handle:
     _snake_case    :  Dict					=					pickle.load(lowerCAmelCase)

    _snake_case    :  Optional[int]					=					tokenizer_new.tokenize(lowerCAmelCase)

    self.assertListEqual(lowerCAmelCase				,		lowerCAmelCase)

   def      UpperCamelCase_  (					self							:			str)				->				int:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					MecabTokenizer(mecab_dic="""ipadic""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[int])				->				Union[str, Any]:

    """simple docstring"""







    try:
     _snake_case    :  Optional[int]					=					MecabTokenizer(mecab_dic="""unidic_lite""")
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			List[str])				->				Union[str, Any]:

    """simple docstring"""







    try:
     _snake_case    :  List[Any]					=					MecabTokenizer(mecab_dic="""unidic""")
    except ModuleNotFoundError:
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				List[Any]:

    """simple docstring"""







    _snake_case    :  List[str]					=					MecabTokenizer(do_lower_case=lowerCAmelCase				,		mecab_dic="""ipadic""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    try:
     _snake_case    :  Dict					=					MecabTokenizer(
         do_lower_case=lowerCAmelCase				,		normalize_text=lowerCAmelCase				,		mecab_option="""-d /usr/local/lib/mecab/dic/jumandic""")
    except RuntimeError:
     # if dict doesn't exist in the system, previous code raises this error.
     return

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				Tuple:

    """simple docstring"""







    _snake_case    :  str					=					MecabTokenizer(normalize_text=lowerCAmelCase				,		mecab_dic="""ipadic""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				str:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					self.tokenizer_class(self.vocab_file				,		word_tokenizer_type="""sudachi""")
    self.assertIsNotNone(lowerCAmelCase)

    _snake_case    :  List[str]					=					"""こんにちは、世界。\nこんばんは、世界。"""
    _snake_case    :  str					=					tokenizer.tokenize(lowerCAmelCase)
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

    _snake_case    :  Optional[Any]					=					os.path.join(self.tmpdirname				,		"""tokenizer.bin""")
    with open(lowerCAmelCase				,		"""wb""") as handle:
     pickle.dump(lowerCAmelCase				,		lowerCAmelCase)

    with open(lowerCAmelCase				,		"""rb""") as handle:
     _snake_case    :  Optional[Any]					=					pickle.load(lowerCAmelCase)

    _snake_case    :  Tuple					=					tokenizer_new.tokenize(lowerCAmelCase)

    self.assertListEqual(lowerCAmelCase				,		lowerCAmelCase)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			List[Any])				->				List[str]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					SudachiTokenizer(sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			str)				->				Tuple:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					SudachiTokenizer(sudachi_dict_type="""core"""				,		sudachi_split_mode="""A""")

    self.assertListEqual(tokenizer.tokenize("""外国人参政権""")				,		["""外国""", """人""", """参政""", """権"""])

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				List[str]:

    """simple docstring"""







    _snake_case    :  Dict					=					SudachiTokenizer(sudachi_dict_type="""core"""				,		sudachi_split_mode="""B""")

    self.assertListEqual(tokenizer.tokenize("""外国人参政権""")				,		["""外国人""", """参政権"""])

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Dict)				->				str:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					SudachiTokenizer(sudachi_dict_type="""core"""				,		sudachi_split_mode="""C""")

    self.assertListEqual(tokenizer.tokenize("""外国人参政権""")				,		["""外国人参政権"""])

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Tuple)				->				Tuple:

    """simple docstring"""







    _snake_case    :  List[str]					=					SudachiTokenizer(do_lower_case=lowerCAmelCase				,		sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Optional[int])				->				Union[str, Any]:

    """simple docstring"""







    _snake_case    :  Dict					=					SudachiTokenizer(normalize_text=lowerCAmelCase				,		sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]				,		)

   @require_sudachi
   def      UpperCamelCase_  (					self							:			Any)				->				Union[str, Any]:

    """simple docstring"""







    _snake_case    :  Tuple					=					SudachiTokenizer(trim_whitespace=lowerCAmelCase				,		sudachi_dict_type="""core""")

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Any)				->				Any:

    """simple docstring"""







    _snake_case    :  int					=					self.tokenizer_class(self.vocab_file				,		word_tokenizer_type="""jumanpp""")
    self.assertIsNotNone(lowerCAmelCase)

    _snake_case    :  Optional[Any]					=					"""こんにちは、世界。\nこんばんは、世界。"""
    _snake_case    :  Tuple					=					tokenizer.tokenize(lowerCAmelCase)
    self.assertListEqual(lowerCAmelCase				,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
    self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 12, 10, 14, 4, 9, 12, 10, 14])

    _snake_case    :  str					=					os.path.join(self.tmpdirname				,		"""tokenizer.bin""")
    with open(lowerCAmelCase				,		"""wb""") as handle:
     pickle.dump(lowerCAmelCase				,		lowerCAmelCase)

    with open(lowerCAmelCase				,		"""rb""") as handle:
     _snake_case    :  int					=					pickle.load(lowerCAmelCase)

    _snake_case    :  List[Any]					=					tokenizer_new.tokenize(lowerCAmelCase)

    self.assertListEqual(lowerCAmelCase				,		lowerCAmelCase)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				List[Any]:

    """simple docstring"""







    _snake_case    :  Optional[Any]					=					JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Dict)				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					JumanppTokenizer(do_lower_case=lowerCAmelCase)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			int)				->				List[str]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					JumanppTokenizer(normalize_text=lowerCAmelCase)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Optional[int])				->				List[str]:

    """simple docstring"""







    _snake_case    :  str					=					JumanppTokenizer(trim_whitespace=lowerCAmelCase)

    self.assertListEqual(
        tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)				,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]				,		)

   @require_jumanpp
   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					JumanppTokenizer()

    self.assertListEqual(
        tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。""")				,		["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]				,		)

   def      UpperCamelCase_  (					self							:			List[str])				->				Optional[Any]:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

    _snake_case    :  str					=					{}
    for i, token in enumerate(lowerCAmelCase):
     _snake_case    :  List[Any]					=					i
    _snake_case    :  List[Any]					=					WordpieceTokenizer(vocab=lowerCAmelCase				,		unk_token="""[UNK]""")

    self.assertListEqual(tokenizer.tokenize("""""")				,		[])

    self.assertListEqual(tokenizer.tokenize("""こんにちは""")				,		["""こんにちは"""])

    self.assertListEqual(tokenizer.tokenize("""こんばんは""")				,		["""こん""", """##ばんは"""])

    self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは""")				,		["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""])

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				str:

    """simple docstring"""







    _snake_case    :  Optional[int]					=					BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp""")
    _snake_case    :  Tuple					=					tokenizer.subword_tokenizer

    _snake_case    :  Tuple					=					subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。""")
    self.assertListEqual(lowerCAmelCase				,		["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""])

    _snake_case    :  Union[str, Any]					=					subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは""")
    self.assertListEqual(lowerCAmelCase				,		["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""])


   def      UpperCamelCase_  (					self							:			Union[str, Any])				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  int					=					self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese""")

    _snake_case    :  str					=					tokenizer.encode("""ありがとう。"""				,		add_special_tokens=lowerCAmelCase)
    _snake_case    :  Optional[Any]					=					tokenizer.encode("""どういたしまして。"""				,		add_special_tokens=lowerCAmelCase)

    _snake_case    :  str					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase)
    _snake_case    :  Dict					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase				,		lowerCAmelCase)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 		snake_case						(	SCREAMING_SNAKE_CASE_					,unittest.TestCase  ):



   '''simple docstring'''






   snake_case_					: Tuple          =		BertJapaneseTokenizer
   snake_case_					: Dict          =		False

   def      UpperCamelCase_  (					self							:			Optional[Any])				->				List[Any]:

    """simple docstring"""







    super().setUp()

    _snake_case    :  Optional[Any]					=					["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

    _snake_case    :  Optional[Any]					=					os.path.join(self.tmpdirname				,		VOCAB_FILES_NAMES["""vocab_file"""])
    with open(self.vocab_file				,		"""w"""				,		encoding="""utf-8""") as vocab_writer:
     vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))

   def      UpperCamelCase_  (					self							:			str				,		**lowerCAmelCase							:			Union[str, Any])				->				Any:

    """simple docstring"""







    return BertJapaneseTokenizer.from_pretrained(self.tmpdirname				,		subword_tokenizer_type="""character"""				,		**lowerCAmelCase)

   def      UpperCamelCase_  (					self							:			str				,		lowerCAmelCase							:			Tuple)				->				Optional[int]:

    """simple docstring"""







    _snake_case    :  Any					=					"""こんにちは、世界。 \nこんばんは、世界。"""
    _snake_case    :  List[Any]					=					"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
    return input_text, output_text

   def      UpperCamelCase_  (					self							:			str)				->				int:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			List[str])				->				Optional[Any]:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			str)				->				Any:

    """simple docstring"""







    pass  # TODO add if relevant

   def      UpperCamelCase_  (					self							:			Optional[int])				->				int:

    """simple docstring"""







    _snake_case    :  Union[str, Any]					=					self.tokenizer_class(self.vocab_file				,		subword_tokenizer_type="""character""")

    _snake_case    :  Dict					=					tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。""")
    self.assertListEqual(
        lowerCAmelCase				,		["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""])
    self.assertListEqual(
        tokenizer.convert_tokens_to_ids(lowerCAmelCase)				,		[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])

   def      UpperCamelCase_  (					self							:			List[str])				->				int:

    """simple docstring"""







    _snake_case    :  str					=					["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

    _snake_case    :  int					=					{}
    for i, token in enumerate(lowerCAmelCase):
     _snake_case    :  int					=					i
    _snake_case    :  Optional[int]					=					CharacterTokenizer(vocab=lowerCAmelCase				,		unk_token="""[UNK]""")

    self.assertListEqual(tokenizer.tokenize("""""")				,		[])

    self.assertListEqual(tokenizer.tokenize("""こんにちは""")				,		["""こ""", """ん""", """に""", """ち""", """は"""])

    self.assertListEqual(tokenizer.tokenize("""こんにちほ""")				,		["""こ""", """ん""", """に""", """ち""", """[UNK]"""])


   def      UpperCamelCase_  (					self							:			int)				->				List[str]:

    """simple docstring"""







    _snake_case    :  Optional[int]					=					self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char""")

    _snake_case    :  List[str]					=					tokenizer.encode("""ありがとう。"""				,		add_special_tokens=lowerCAmelCase)
    _snake_case    :  Optional[int]					=					tokenizer.encode("""どういたしまして。"""				,		add_special_tokens=lowerCAmelCase)

    _snake_case    :  str					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase)
    _snake_case    :  Any					=					tokenizer.build_inputs_with_special_tokens(lowerCAmelCase				,		lowerCAmelCase)

    # 2 is for "[CLS]", 3 is for "[SEP]"
    assert encoded_sentence == [2] + text + [3]
    assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class 		snake_case						(	unittest.TestCase  ):



   '''simple docstring'''






   def      UpperCamelCase_  (					self							:			int)				->				List[Any]:

    """simple docstring"""







    _snake_case    :  List[str]					=					"""cl-tohoku/bert-base-japanese"""
    _snake_case    :  int					=					AutoTokenizer.from_pretrained(lowerCAmelCase)
    self.assertIsInstance(lowerCAmelCase				,		lowerCAmelCase)







class 		snake_case						(	unittest.TestCase  ):



   '''simple docstring'''






   def      UpperCamelCase_  (					self							:			str)				->				Any:

    """simple docstring"""







    _snake_case    :  str					=					"""cl-tohoku/bert-base-japanese"""
    with self.assertLogs("""transformers"""				,		level="""WARNING""") as cm:
     BertTokenizer.from_pretrained(lowerCAmelCase)
     self.assertTrue(
         cm.records[0].message.startswith(
             """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
             """ is called from."""))
    _snake_case    :  Any					=					"""bert-base-cased"""
    with self.assertLogs("""transformers"""				,		level="""WARNING""") as cm:
     BertJapaneseTokenizer.from_pretrained(lowerCAmelCase)
     self.assertTrue(
         cm.records[0].message.startswith(
             """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
             """ is called from."""))





import sys

a__    =	(
    """73167176531330624919225119674426574742355349194934"""
    """96983520312774506326239578318016984801869478851843"""
    """85861560789112949495459501737958331952853208805511"""
    """12540698747158523863050715693290963295227443043557"""
    """66896648950445244523161731856403098711121722383113"""
    """62229893423380308135336276614282806444486645238749"""
    """30358907296290491560440772390713810515859307960866"""
    """70172427121883998797908792274921901699720888093776"""
    """65727333001053367881220235421809751254540594752243"""
    """52584907711670556013604839586446706324415722155397"""
    """53697817977846174064955149290862569321978468622482"""
    """83972241375657056057490261407972968652414535100474"""
    """82166370484403199890008895243450658541227588666881"""
    """16427171479924442928230863465674813919123162824586"""
    """17866458359124566529476545682848912883142607690042"""
    """24219022671055626321111109370544217506941658960408"""
    """07198403850962455444362981230987879927244284909188"""
    """84580156166097919133875499200524063689912560717606"""
    """05886116467109405077541002256983155200055935729725"""
    """71636269561882670428252483600823257530420752963450"""
)



def       lowercase       (   SCREAMING_SNAKE_CASE__       :	str = N				)  ->     int:
 _snake_case    :  int					=					-sys.maxsize - 1
 for i in range(len(SCREAMING_SNAKE_CASE__				) - 12				):
  _snake_case    :  Optional[int]					=					1
  for j in range(13				):
   product *= int(n[i + j]				)
  if product > largest_product:
   _snake_case    :  List[str]					=					product
 return largest_product


if __name__ == "__main__":
  print(F'''{solution() = }''')
