


'''simple docstring'''






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class   __snake_case			(  _SCREAMING_SNAKE_CASE      ,unittest.TestCase):







    """simple docstring"""



    lowercase  					=   BertJapaneseTokenizer
    lowercase  					=   False
    lowercase  					=   True

    def __lowercase  (  self       :    List[Any]						)  ->    int:
      super().setUp()

      lowerCAmelCase_     : int     =  [
          """[UNK]""",
          """[CLS]""",
          """[SEP]""",
          """こんにちは""",
          """こん""",
          """にちは""",
          """ばんは""",
          """##こん""",
          """##にちは""",
          """##ばんは""",
          """世界""",
          """##世界""",
          """、""",
          """##、""",
          """。""",
          """##。""",
      ]

      lowerCAmelCase_     : Tuple     =  os.path.join(self.tmpdirname       ,		VOCAB_FILES_NAMES["""vocab_file"""]						)
      with open(self.vocab_file       ,		"""w"""       ,		encoding="""utf-8"""						) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)

    def __lowercase  (  self       :    Optional[int]       ,		lowerCamelCase       :    Optional[Any]						)  ->    List[Any]:
      lowerCAmelCase_     : str     =  """こんにちは、世界。 \nこんばんは、世界。"""
      lowerCAmelCase_     : Union[str, Any]     =  """こんにちは 、 世界 。 こんばんは 、 世界 。"""
      return input_text, output_text

    def __lowercase  (  self       :    Union[str, Any]       ,		lowerCamelCase       :    List[str]						)  ->    Tuple:
      lowerCAmelCase_,				lowerCAmelCase_     : Union[str, Any]     =  self.get_input_output_texts(lowerCamelCase						)
      lowerCAmelCase_     : Tuple     =  tokenizer.encode(lowerCamelCase       ,		add_special_tokens=lowerCamelCase						)
      lowerCAmelCase_     : str     =  tokenizer.decode(lowerCamelCase       ,		clean_up_tokenization_spaces=lowerCamelCase						)
      return text, ids

    def __lowercase  (  self       :    List[Any]						)  ->    int:
      pass  # TODO add if relevant

    def __lowercase  (  self       :    Any						)  ->    Union[str, Any]:
      pass  # TODO add if relevant

    def __lowercase  (  self       :    List[Any]						)  ->    str:
      pass  # TODO add if relevant

    def __lowercase  (  self       :    Dict						)  ->    Optional[Any]:
      lowerCAmelCase_     : Dict     =  self.tokenizer_class(self.vocab_file						)

      lowerCAmelCase_     : int     =  tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""						)
      self.assertListEqual(lowerCamelCase       ,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,		[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

    def __lowercase  (  self       :    Union[str, Any]						)  ->    Any:
      lowerCAmelCase_     : List[str]     =  self.tokenizer_class(self.vocab_file       ,		word_tokenizer_type="""mecab"""						)
      self.assertIsNotNone(lowerCamelCase						)

      lowerCAmelCase_     : List[Any]     =  """こんにちは、世界。\nこんばんは、世界。"""
      lowerCAmelCase_     : Union[str, Any]     =  tokenizer.tokenize(lowerCamelCase						)
      self.assertListEqual(lowerCamelCase       ,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,		[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

      lowerCAmelCase_     : Union[str, Any]     =  os.path.join(self.tmpdirname       ,		"""tokenizer.bin"""						)
      with open(lowerCamelCase       ,		"""wb"""						) as handle:
        pickle.dump(lowerCamelCase       ,		lowerCamelCase						)

      with open(lowerCamelCase       ,		"""rb"""						) as handle:
        lowerCAmelCase_     : Any     =  pickle.load(lowerCamelCase						)

      lowerCAmelCase_     : int     =  tokenizer_new.tokenize(lowerCamelCase						)

      self.assertListEqual(lowerCamelCase       ,		lowerCamelCase						)

    def __lowercase  (  self       :    int						)  ->    Dict:
      lowerCAmelCase_     : Optional[int]     =  MecabTokenizer(mecab_dic="""ipadic"""						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,		)

    def __lowercase  (  self       :    List[str]						)  ->    List[Any]:
      try:
        lowerCAmelCase_     : int     =  MecabTokenizer(mecab_dic="""unidic_lite"""						)
      except ModuleNotFoundError:
        return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,		)

    def __lowercase  (  self       :    Optional[Any]						)  ->    Tuple:
      try:
        lowerCAmelCase_     : Dict     =  MecabTokenizer(mecab_dic="""unidic"""						)
      except ModuleNotFoundError:
        return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,		)

    def __lowercase  (  self       :    Dict						)  ->    Optional[int]:
      lowerCAmelCase_     : Dict     =  MecabTokenizer(do_lower_case=lowerCamelCase       ,		mecab_dic="""ipadic"""						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,		)

    def __lowercase  (  self       :    Tuple						)  ->    int:
      try:
        lowerCAmelCase_     : Any     =  MecabTokenizer(
            do_lower_case=lowerCamelCase       ,		normalize_text=lowerCamelCase       ,		mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""						)
      except RuntimeError:
        # if dict doesn't exist in the system, previous code raises this error.
        return

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,		)

    def __lowercase  (  self       :    Dict						)  ->    Tuple:
      lowerCAmelCase_     : Any     =  MecabTokenizer(normalize_text=lowerCamelCase       ,		mecab_dic="""ipadic"""						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]       ,		)

    @require_sudachi
    def __lowercase  (  self       :    List[Any]						)  ->    Tuple:
      lowerCAmelCase_     : List[str]     =  self.tokenizer_class(self.vocab_file       ,		word_tokenizer_type="""sudachi"""						)
      self.assertIsNotNone(lowerCamelCase						)

      lowerCAmelCase_     : Union[str, Any]     =  """こんにちは、世界。\nこんばんは、世界。"""
      lowerCAmelCase_     : Optional[int]     =  tokenizer.tokenize(lowerCamelCase						)
      self.assertListEqual(lowerCamelCase       ,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,		[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

      lowerCAmelCase_     : Optional[Any]     =  os.path.join(self.tmpdirname       ,		"""tokenizer.bin"""						)
      with open(lowerCamelCase       ,		"""wb"""						) as handle:
        pickle.dump(lowerCamelCase       ,		lowerCamelCase						)

      with open(lowerCamelCase       ,		"""rb"""						) as handle:
        lowerCAmelCase_     : str     =  pickle.load(lowerCamelCase						)

      lowerCAmelCase_     : List[Any]     =  tokenizer_new.tokenize(lowerCamelCase						)

      self.assertListEqual(lowerCamelCase       ,		lowerCamelCase						)

    @require_sudachi
    def __lowercase  (  self       :    Dict						)  ->    Optional[Any]:
      lowerCAmelCase_     : Optional[int]     =  SudachiTokenizer(sudachi_dict_type="""core"""						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]       ,		)

    @require_sudachi
    def __lowercase  (  self       :    Union[str, Any]						)  ->    Union[str, Any]:
      lowerCAmelCase_     : List[Any]     =  SudachiTokenizer(sudachi_dict_type="""core"""       ,		sudachi_split_mode="""A"""						)

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)       ,		["""外国""", """人""", """参政""", """権"""]						)

    @require_sudachi
    def __lowercase  (  self       :    Union[str, Any]						)  ->    Union[str, Any]:
      lowerCAmelCase_     : Tuple     =  SudachiTokenizer(sudachi_dict_type="""core"""       ,		sudachi_split_mode="""B"""						)

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)       ,		["""外国人""", """参政権"""]						)

    @require_sudachi
    def __lowercase  (  self       :    List[str]						)  ->    Dict:
      lowerCAmelCase_     : int     =  SudachiTokenizer(sudachi_dict_type="""core"""       ,		sudachi_split_mode="""C"""						)

      self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)       ,		["""外国人参政権"""]						)

    @require_sudachi
    def __lowercase  (  self       :    Any						)  ->    List[Any]:
      lowerCAmelCase_     : Dict     =  SudachiTokenizer(do_lower_case=lowerCamelCase       ,		sudachi_dict_type="""core"""						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]       ,		)

    @require_sudachi
    def __lowercase  (  self       :    List[Any]						)  ->    str:
      lowerCAmelCase_     : List[Any]     =  SudachiTokenizer(normalize_text=lowerCamelCase       ,		sudachi_dict_type="""core"""						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]       ,		)

    @require_sudachi
    def __lowercase  (  self       :    Optional[Any]						)  ->    Optional[int]:
      lowerCAmelCase_     : Any     =  SudachiTokenizer(trim_whitespace=lowerCamelCase       ,		sudachi_dict_type="""core"""						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,		)

    @require_jumanpp
    def __lowercase  (  self       :    Any						)  ->    List[Any]:
      lowerCAmelCase_     : Optional[int]     =  self.tokenizer_class(self.vocab_file       ,		word_tokenizer_type="""jumanpp"""						)
      self.assertIsNotNone(lowerCamelCase						)

      lowerCAmelCase_     : List[Any]     =  """こんにちは、世界。\nこんばんは、世界。"""
      lowerCAmelCase_     : Union[str, Any]     =  tokenizer.tokenize(lowerCamelCase						)
      self.assertListEqual(lowerCamelCase       ,		["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
      self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,		[3, 12, 10, 14, 4, 9, 12, 10, 14]						)

      lowerCAmelCase_     : Tuple     =  os.path.join(self.tmpdirname       ,		"""tokenizer.bin"""						)
      with open(lowerCamelCase       ,		"""wb"""						) as handle:
        pickle.dump(lowerCamelCase       ,		lowerCamelCase						)

      with open(lowerCamelCase       ,		"""rb"""						) as handle:
        lowerCAmelCase_     : Tuple     =  pickle.load(lowerCamelCase						)

      lowerCAmelCase_     : List[str]     =  tokenizer_new.tokenize(lowerCamelCase						)

      self.assertListEqual(lowerCamelCase       ,		lowerCamelCase						)

    @require_jumanpp
    def __lowercase  (  self       :    Dict						)  ->    Dict:
      lowerCAmelCase_     : Union[str, Any]     =  JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,		)

    @require_jumanpp
    def __lowercase  (  self       :    Optional[Any]						)  ->    int:
      lowerCAmelCase_     : List[str]     =  JumanppTokenizer(do_lower_case=lowerCamelCase						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,		)

    @require_jumanpp
    def __lowercase  (  self       :    int						)  ->    Any:
      lowerCAmelCase_     : Dict     =  JumanppTokenizer(normalize_text=lowerCamelCase						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,		)

    @require_jumanpp
    def __lowercase  (  self       :    Tuple						)  ->    str:
      lowerCAmelCase_     : Any     =  JumanppTokenizer(trim_whitespace=lowerCamelCase						)

      self.assertListEqual(
          tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)       ,		["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]       ,		)

    @require_jumanpp
    def __lowercase  (  self       :    Dict						)  ->    Tuple:
      lowerCAmelCase_     : int     =  JumanppTokenizer()

      self.assertListEqual(
          tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""						)       ,		["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]       ,		)

    def __lowercase  (  self       :    int						)  ->    Optional[Any]:
      lowerCAmelCase_     : Tuple     =  ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

      lowerCAmelCase_     : Optional[int]     =  {}
      for i, token in enumerate(lowerCamelCase						):
        lowerCAmelCase_     : int     =  i
      lowerCAmelCase_     : Optional[int]     =  WordpieceTokenizer(vocab=lowerCamelCase       ,		unk_token="""[UNK]"""						)

      self.assertListEqual(tokenizer.tokenize(""""""						)       ,		[]						)

      self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)       ,		["""こんにちは"""]						)

      self.assertListEqual(tokenizer.tokenize("""こんばんは"""						)       ,		["""こん""", """##ばんは"""]						)

      self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""						)       ,		["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]						)

    def __lowercase  (  self       :    str						)  ->    Dict:
      lowerCAmelCase_     : Tuple     =  BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""						)
      lowerCAmelCase_     : Tuple     =  tokenizer.subword_tokenizer

      lowerCAmelCase_     : Any     =  subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""						)
      self.assertListEqual(lowerCamelCase       ,		["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]						)

      lowerCAmelCase_     : List[str]     =  subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""						)
      self.assertListEqual(lowerCamelCase       ,		["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]						)





    def __lowercase  (  self       :    str						)  ->    str:
      lowerCAmelCase_     : str     =  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""						)

      lowerCAmelCase_     : int     =  tokenizer.encode("""ありがとう。"""       ,		add_special_tokens=lowerCamelCase						)
      lowerCAmelCase_     : str     =  tokenizer.encode("""どういたしまして。"""       ,		add_special_tokens=lowerCamelCase						)

      lowerCAmelCase_     : Any     =  tokenizer.build_inputs_with_special_tokens(lowerCamelCase						)
      lowerCAmelCase_     : Optional[Any]     =  tokenizer.build_inputs_with_special_tokens(lowerCamelCase       ,		lowerCamelCase						)

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class   __snake_case			(  _SCREAMING_SNAKE_CASE      ,unittest.TestCase):







    """simple docstring"""



    lowercase  					=   BertJapaneseTokenizer
    lowercase  					=   False

    def __lowercase  (  self       :    List[Any]						)  ->    str:
      super().setUp()

      lowerCAmelCase_     : Tuple     =  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

      lowerCAmelCase_     : Optional[Any]     =  os.path.join(self.tmpdirname       ,		VOCAB_FILES_NAMES["""vocab_file"""]						)
      with open(self.vocab_file       ,		"""w"""       ,		encoding="""utf-8"""						) as vocab_writer:
        vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)

    def __lowercase  (  self       :    List[str]       ,		**lowerCamelCase       :    Dict						)  ->    Dict:
      return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,		subword_tokenizer_type="""character"""       ,		**lowerCamelCase						)

    def __lowercase  (  self       :    str       ,		lowerCamelCase       :    Optional[int]						)  ->    Optional[Any]:
      lowerCAmelCase_     : Dict     =  """こんにちは、世界。 \nこんばんは、世界。"""
      lowerCAmelCase_     : Dict     =  """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
      return input_text, output_text

    def __lowercase  (  self       :    Tuple						)  ->    Dict:
      pass  # TODO add if relevant

    def __lowercase  (  self       :    Dict						)  ->    str:
      pass  # TODO add if relevant

    def __lowercase  (  self       :    List[str]						)  ->    List[str]:
      pass  # TODO add if relevant

    def __lowercase  (  self       :    Dict						)  ->    int:
      lowerCAmelCase_     : Tuple     =  self.tokenizer_class(self.vocab_file       ,		subword_tokenizer_type="""character"""						)

      lowerCAmelCase_     : Dict     =  tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""						)
      self.assertListEqual(
          lowerCamelCase       ,		["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]						)
      self.assertListEqual(
          tokenizer.convert_tokens_to_ids(lowerCamelCase						)       ,		[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)

    def __lowercase  (  self       :    Optional[Any]						)  ->    Dict:
      lowerCAmelCase_     : Tuple     =  ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

      lowerCAmelCase_     : Tuple     =  {}
      for i, token in enumerate(lowerCamelCase						):
        lowerCAmelCase_     : Dict     =  i
      lowerCAmelCase_     : Dict     =  CharacterTokenizer(vocab=lowerCamelCase       ,		unk_token="""[UNK]"""						)

      self.assertListEqual(tokenizer.tokenize(""""""						)       ,		[]						)

      self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)       ,		["""こ""", """ん""", """に""", """ち""", """は"""]						)

      self.assertListEqual(tokenizer.tokenize("""こんにちほ"""						)       ,		["""こ""", """ん""", """に""", """ち""", """[UNK]"""]						)





    def __lowercase  (  self       :    Optional[int]						)  ->    int:
      lowerCAmelCase_     : Optional[int]     =  self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""						)

      lowerCAmelCase_     : Optional[Any]     =  tokenizer.encode("""ありがとう。"""       ,		add_special_tokens=lowerCamelCase						)
      lowerCAmelCase_     : Optional[int]     =  tokenizer.encode("""どういたしまして。"""       ,		add_special_tokens=lowerCamelCase						)

      lowerCAmelCase_     : int     =  tokenizer.build_inputs_with_special_tokens(lowerCamelCase						)
      lowerCAmelCase_     : List[Any]     =  tokenizer.build_inputs_with_special_tokens(lowerCamelCase       ,		lowerCamelCase						)

      # 2 is for "[CLS]", 3 is for "[SEP]"
      assert encoded_sentence == [2] + text + [3]
      assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class   __snake_case			(  unittest.TestCase):







    """simple docstring"""



    def __lowercase  (  self       :    Optional[Any]						)  ->    List[Any]:
      lowerCAmelCase_     : int     =  """cl-tohoku/bert-base-japanese"""
      lowerCAmelCase_     : Optional[Any]     =  AutoTokenizer.from_pretrained(lowerCamelCase						)
      self.assertIsInstance(lowerCamelCase       ,		lowerCamelCase						)







class   __snake_case			(  unittest.TestCase):







    """simple docstring"""



    def __lowercase  (  self       :    str						)  ->    Optional[Any]:
      lowerCAmelCase_     : Optional[Any]     =  """cl-tohoku/bert-base-japanese"""
      with self.assertLogs("""transformers"""       ,		level="""WARNING"""						) as cm:
        BertTokenizer.from_pretrained(lowerCamelCase						)
        self.assertTrue(
            cm.records[0].message.startswith(
                """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                """ is called from."""						)						)
      lowerCAmelCase_     : Optional[Any]     =  """bert-base-cased"""
      with self.assertLogs("""transformers"""       ,		level="""WARNING"""						) as cm:
        BertJapaneseTokenizer.from_pretrained(lowerCamelCase						)
        self.assertTrue(
            cm.records[0].message.startswith(
                """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                """ is called from."""						)						)







'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


__A   :    str											= {
    "configuration_roc_bert": ["ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "RoCBertConfig"],
    "tokenization_roc_bert": ["RoCBertTokenizer"],
}

try:
  if not is_tokenizers_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  pass

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  __A   :    Optional[int]											= [
      "ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
      "RoCBertForCausalLM",
      "RoCBertForMaskedLM",
      "RoCBertForMultipleChoice",
      "RoCBertForPreTraining",
      "RoCBertForQuestionAnswering",
      "RoCBertForSequenceClassification",
      "RoCBertForTokenClassification",
      "RoCBertLayer",
      "RoCBertModel",
      "RoCBertPreTrainedModel",
      "load_tf_weights_in_roc_bert",
  ]

if TYPE_CHECKING:
  from .configuration_roc_bert import ROC_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, RoCBertConfig
  from .tokenization_roc_bert import RoCBertTokenizer

  try:
    if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    raise OptionalDependencyNotAvailable()

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_roc_bert import (
        ROC_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
        RoCBertForCausalLM,
        RoCBertForMaskedLM,
        RoCBertForMultipleChoice,
        RoCBertForPreTraining,
        RoCBertForQuestionAnswering,
        RoCBertForSequenceClassification,
        RoCBertForTokenClassification,
        RoCBertLayer,
        RoCBertModel,
        RoCBertPreTrainedModel,
        load_tf_weights_in_roc_bert,
    )


else:
  import sys

  __A   :    Optional[int]											= _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



