

"""simple docstring"""




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class   __lowercase      (     __lowerCamelCase						,      unittest.TestCase       ):
  snake_case_												=							BertJapaneseTokenizer
  snake_case_												=							False
  snake_case_												=							True

  def  __lowercase							(	self		:      Any     ):






   '''simple docstring'''
   super().setUp()

   UpperCAmelCase__       :	int        =       [
       """[UNK]""",
       """[CLS]""",
       """[SEP]""",
       """こんにちは""",
       """こん""",
       """にちは""",
       """ばんは""",
       """##こん""",
       """##にちは""",
       """##ばんは""",
       """世界""",
       """##世界""",
       """、""",
       """##、""",
       """。""",
       """##。""",
   ]

   UpperCAmelCase__       :	Optional[Any]        =       os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES["""vocab_file"""]     )
   with open(self.vocab_file       ,"""w"""       ,encoding="""utf-8"""     ) as vocab_writer:
    vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )

  def  __lowercase							(	self		:      Dict       ,A		:      List[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[Any]        =       """こんにちは、世界。 \nこんばんは、世界。"""
   UpperCAmelCase__       :	List[str]        =       """こんにちは 、 世界 。 こんばんは 、 世界 。"""
   return input_text, output_text

  def  __lowercase							(	self		:      Tuple       ,A		:      Tuple     ):






   '''simple docstring'''
   UpperCAmelCase__ ,   UpperCAmelCase__       :	Optional[int]        =       self.get_input_output_texts(A     )
   UpperCAmelCase__       :	List[str]        =       tokenizer.encode(A       ,add_special_tokens=A     )
   UpperCAmelCase__       :	Optional[Any]        =       tokenizer.decode(A       ,clean_up_tokenization_spaces=A     )
   return text, ids

  def  __lowercase							(	self		:      Optional[Any]     ):






   '''simple docstring'''
   pass  # TODO add if relevant

  def  __lowercase							(	self		:      Optional[Any]     ):






   '''simple docstring'''
   pass  # TODO add if relevant

  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   pass  # TODO add if relevant

  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Union[str, Any]        =       self.tokenizer_class(self.vocab_file     )

   UpperCAmelCase__       :	Any        =       tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""     )
   self.assertListEqual(A       ,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(A     )       ,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )

  def  __lowercase							(	self		:      Optional[int]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[Any]        =       self.tokenizer_class(self.vocab_file       ,word_tokenizer_type="""mecab"""     )
   self.assertIsNotNone(A     )

   UpperCAmelCase__       :	int        =       """こんにちは、世界。\nこんばんは、世界。"""
   UpperCAmelCase__       :	Tuple        =       tokenizer.tokenize(A     )
   self.assertListEqual(A       ,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(A     )       ,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )

   UpperCAmelCase__       :	List[Any]        =       os.path.join(self.tmpdirname       ,"""tokenizer.bin"""     )
   with open(A       ,"""wb"""     ) as handle:
    pickle.dump(A       ,A     )

   with open(A       ,"""rb"""     ) as handle:
    UpperCAmelCase__       :	Tuple        =       pickle.load(A     )

   UpperCAmelCase__       :	Optional[Any]        =       tokenizer_new.tokenize(A     )

   self.assertListEqual(A       ,A     )

  def  __lowercase							(	self		:      Optional[int]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Any        =       MecabTokenizer(mecab_dic="""ipadic"""     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,)

  def  __lowercase							(	self		:      Optional[Any]     ):






   '''simple docstring'''
   try:
    UpperCAmelCase__       :	Any        =       MecabTokenizer(mecab_dic="""unidic_lite"""     )
   except ModuleNotFoundError:
    return

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,)

  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   try:
    UpperCAmelCase__       :	Optional[Any]        =       MecabTokenizer(mecab_dic="""unidic"""     )
   except ModuleNotFoundError:
    return

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,)

  def  __lowercase							(	self		:      Any     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Union[str, Any]        =       MecabTokenizer(do_lower_case=A       ,mecab_dic="""ipadic"""     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,)

  def  __lowercase							(	self		:      Union[str, Any]     ):






   '''simple docstring'''
   try:
    UpperCAmelCase__       :	Tuple        =       MecabTokenizer(
        do_lower_case=A       ,normalize_text=A       ,mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""     )
   except RuntimeError:
    # if dict doesn't exist in the system, previous code raises this error.
    return

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,)

  def  __lowercase							(	self		:      Union[str, Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Union[str, Any]        =       MecabTokenizer(normalize_text=A       ,mecab_dic="""ipadic"""     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]       ,)

  @require_sudachi
  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Tuple        =       self.tokenizer_class(self.vocab_file       ,word_tokenizer_type="""sudachi"""     )
   self.assertIsNotNone(A     )

   UpperCAmelCase__       :	Optional[Any]        =       """こんにちは、世界。\nこんばんは、世界。"""
   UpperCAmelCase__       :	Any        =       tokenizer.tokenize(A     )
   self.assertListEqual(A       ,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(A     )       ,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )

   UpperCAmelCase__       :	List[Any]        =       os.path.join(self.tmpdirname       ,"""tokenizer.bin"""     )
   with open(A       ,"""wb"""     ) as handle:
    pickle.dump(A       ,A     )

   with open(A       ,"""rb"""     ) as handle:
    UpperCAmelCase__       :	int        =       pickle.load(A     )

   UpperCAmelCase__       :	Any        =       tokenizer_new.tokenize(A     )

   self.assertListEqual(A       ,A     )

  @require_sudachi
  def  __lowercase							(	self		:      List[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	str        =       SudachiTokenizer(sudachi_dict_type="""core"""     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]       ,)

  @require_sudachi
  def  __lowercase							(	self		:      Tuple     ):






   '''simple docstring'''
   UpperCAmelCase__       :	str        =       SudachiTokenizer(sudachi_dict_type="""core"""       ,sudachi_split_mode="""A"""     )

   self.assertListEqual(tokenizer.tokenize("""外国人参政権"""     )       ,["""外国""", """人""", """参政""", """権"""]     )

  @require_sudachi
  def  __lowercase							(	self		:      Any     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Dict        =       SudachiTokenizer(sudachi_dict_type="""core"""       ,sudachi_split_mode="""B"""     )

   self.assertListEqual(tokenizer.tokenize("""外国人参政権"""     )       ,["""外国人""", """参政権"""]     )

  @require_sudachi
  def  __lowercase							(	self		:      Tuple     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Dict        =       SudachiTokenizer(sudachi_dict_type="""core"""       ,sudachi_split_mode="""C"""     )

   self.assertListEqual(tokenizer.tokenize("""外国人参政権"""     )       ,["""外国人参政権"""]     )

  @require_sudachi
  def  __lowercase							(	self		:      Dict     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       SudachiTokenizer(do_lower_case=A       ,sudachi_dict_type="""core"""     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]       ,)

  @require_sudachi
  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[int]        =       SudachiTokenizer(normalize_text=A       ,sudachi_dict_type="""core"""     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]       ,)

  @require_sudachi
  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[int]        =       SudachiTokenizer(trim_whitespace=A       ,sudachi_dict_type="""core"""     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,)

  @require_jumanpp
  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Dict        =       self.tokenizer_class(self.vocab_file       ,word_tokenizer_type="""jumanpp"""     )
   self.assertIsNotNone(A     )

   UpperCAmelCase__       :	Tuple        =       """こんにちは、世界。\nこんばんは、世界。"""
   UpperCAmelCase__       :	Union[str, Any]        =       tokenizer.tokenize(A     )
   self.assertListEqual(A       ,["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]     )
   self.assertListEqual(tokenizer.convert_tokens_to_ids(A     )       ,[3, 12, 10, 14, 4, 9, 12, 10, 14]     )

   UpperCAmelCase__       :	Optional[int]        =       os.path.join(self.tmpdirname       ,"""tokenizer.bin"""     )
   with open(A       ,"""wb"""     ) as handle:
    pickle.dump(A       ,A     )

   with open(A       ,"""rb"""     ) as handle:
    UpperCAmelCase__       :	Optional[int]        =       pickle.load(A     )

   UpperCAmelCase__       :	Dict        =       tokenizer_new.tokenize(A     )

   self.assertListEqual(A       ,A     )

  @require_jumanpp
  def  __lowercase							(	self		:      Tuple     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Tuple        =       JumanppTokenizer()

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,)

  @require_jumanpp
  def  __lowercase							(	self		:      List[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Tuple        =       JumanppTokenizer(do_lower_case=A     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,)

  @require_jumanpp
  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Optional[int]        =       JumanppTokenizer(normalize_text=A     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,)

  @require_jumanpp
  def  __lowercase							(	self		:      Tuple     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       JumanppTokenizer(trim_whitespace=A     )

   self.assertListEqual(
       tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """     )       ,["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]       ,)

  @require_jumanpp
  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[Any]        =       JumanppTokenizer()

   self.assertListEqual(
       tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""     )       ,["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]       ,)

  def  __lowercase							(	self		:      Tuple     ):






   '''simple docstring'''
   UpperCAmelCase__       :	int        =       ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

   UpperCAmelCase__       :	int        =       {}
   for i, token in enumerate(A     ):
    UpperCAmelCase__       :	List[str]        =       i
   UpperCAmelCase__       :	List[Any]        =       WordpieceTokenizer(vocab=A       ,unk_token="""[UNK]"""     )

   self.assertListEqual(tokenizer.tokenize(""""""     )       ,[]     )

   self.assertListEqual(tokenizer.tokenize("""こんにちは"""     )       ,["""こんにちは"""]     )

   self.assertListEqual(tokenizer.tokenize("""こんばんは"""     )       ,["""こん""", """##ばんは"""]     )

   self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""     )       ,["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]     )

  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	str        =       BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""     )
   UpperCAmelCase__       :	Any        =       tokenizer.subword_tokenizer

   UpperCAmelCase__       :	List[str]        =       subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""     )
   self.assertListEqual(A       ,["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]     )

   UpperCAmelCase__       :	str        =       subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""     )
   self.assertListEqual(A       ,["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]     )




  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[str]        =       self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""     )

   UpperCAmelCase__       :	Tuple        =       tokenizer.encode("""ありがとう。"""       ,add_special_tokens=A     )
   UpperCAmelCase__       :	int        =       tokenizer.encode("""どういたしまして。"""       ,add_special_tokens=A     )

   UpperCAmelCase__       :	Any        =       tokenizer.build_inputs_with_special_tokens(A     )
   UpperCAmelCase__       :	str        =       tokenizer.build_inputs_with_special_tokens(A       ,A     )

   # 2 is for "[CLS]", 3 is for "[SEP]"
   assert encoded_sentence == [2] + text + [3]
   assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class   __lowercase      (     __lowerCamelCase						,      unittest.TestCase       ):
  snake_case_												=							BertJapaneseTokenizer
  snake_case_												=							False

  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   super().setUp()

   UpperCAmelCase__       :	Optional[Any]        =       ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

   UpperCAmelCase__       :	int        =       os.path.join(self.tmpdirname       ,VOCAB_FILES_NAMES["""vocab_file"""]     )
   with open(self.vocab_file       ,"""w"""       ,encoding="""utf-8"""     ) as vocab_writer:
    vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]     )     )

  def  __lowercase							(	self		:      List[Any]       ,**A		:      Optional[Any]     ):






   '''simple docstring'''
   return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,subword_tokenizer_type="""character"""       ,**A     )

  def  __lowercase							(	self		:      Any       ,A		:      List[Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Any        =       """こんにちは、世界。 \nこんばんは、世界。"""
   UpperCAmelCase__       :	Optional[int]        =       """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
   return input_text, output_text

  def  __lowercase							(	self		:      Any     ):






   '''simple docstring'''
   pass  # TODO add if relevant

  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   pass  # TODO add if relevant

  def  __lowercase							(	self		:      List[Any]     ):






   '''simple docstring'''
   pass  # TODO add if relevant

  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[Any]        =       self.tokenizer_class(self.vocab_file       ,subword_tokenizer_type="""character"""     )

   UpperCAmelCase__       :	str        =       tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""     )
   self.assertListEqual(
       A       ,["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]     )
   self.assertListEqual(
       tokenizer.convert_tokens_to_ids(A     )       ,[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]     )

  def  __lowercase							(	self		:      str     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[str]        =       ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

   UpperCAmelCase__       :	Optional[int]        =       {}
   for i, token in enumerate(A     ):
    UpperCAmelCase__       :	List[Any]        =       i
   UpperCAmelCase__       :	List[Any]        =       CharacterTokenizer(vocab=A       ,unk_token="""[UNK]"""     )

   self.assertListEqual(tokenizer.tokenize(""""""     )       ,[]     )

   self.assertListEqual(tokenizer.tokenize("""こんにちは"""     )       ,["""こ""", """ん""", """に""", """ち""", """は"""]     )

   self.assertListEqual(tokenizer.tokenize("""こんにちほ"""     )       ,["""こ""", """ん""", """に""", """ち""", """[UNK]"""]     )




  def  __lowercase							(	self		:      List[str]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	List[Any]        =       self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""     )

   UpperCAmelCase__       :	int        =       tokenizer.encode("""ありがとう。"""       ,add_special_tokens=A     )
   UpperCAmelCase__       :	Dict        =       tokenizer.encode("""どういたしまして。"""       ,add_special_tokens=A     )

   UpperCAmelCase__       :	Any        =       tokenizer.build_inputs_with_special_tokens(A     )
   UpperCAmelCase__       :	Dict        =       tokenizer.build_inputs_with_special_tokens(A       ,A     )

   # 2 is for "[CLS]", 3 is for "[SEP]"
   assert encoded_sentence == [2] + text + [3]
   assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class   __lowercase      (     unittest.TestCase       ):

  def  __lowercase							(	self		:      int     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Dict        =       """cl-tohoku/bert-base-japanese"""
   UpperCAmelCase__       :	str        =       AutoTokenizer.from_pretrained(A     )
   self.assertIsInstance(A       ,A     )







class   __lowercase      (     unittest.TestCase       ):

  def  __lowercase							(	self		:      Union[str, Any]     ):






   '''simple docstring'''
   UpperCAmelCase__       :	Any        =       """cl-tohoku/bert-base-japanese"""
   with self.assertLogs("""transformers"""       ,level="""WARNING"""     ) as cm:
    BertTokenizer.from_pretrained(A     )
    self.assertTrue(
        cm.records[0].message.startswith(
            """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
            """ is called from."""     )     )
   UpperCAmelCase__       :	Dict        =       """bert-base-cased"""
   with self.assertLogs("""transformers"""       ,level="""WARNING"""     ) as cm:
    BertJapaneseTokenizer.from_pretrained(A     )
    self.assertTrue(
        cm.records[0].message.startswith(
            """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
            """ is called from."""     )     )









from typing import List, Optional, Union

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType






class 	__lowercase					(	__snake_case	):
      UpperCamelCase										=							['''image_processor''', '''tokenizer''']
      UpperCamelCase										=							'''BridgeTowerImageProcessor'''
      UpperCamelCase										=							('''RobertaTokenizer''', '''RobertaTokenizerFast''')





      def __init__(		self    :       int       ,							__lowerCamelCase    :       Tuple       ,							__lowerCamelCase    :       List[Any]							)     -> Union[str, Any]:






         """simple docstring"""







         super().__init__(__lowerCamelCase       ,							__lowerCamelCase							)





      def __call__(		self    :       Any       ,							__lowerCamelCase    :       Optional[int]       ,							__lowerCamelCase    :       Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None       ,							__lowerCamelCase    :       bool = True       ,							__lowerCamelCase    :       Union[bool, str, PaddingStrategy] = False       ,							__lowerCamelCase    :       Union[bool, str, TruncationStrategy] = None       ,							__lowerCamelCase    :       Optional[int] = None       ,							__lowerCamelCase    :       int = 0       ,							__lowerCamelCase    :       Optional[int] = None       ,							__lowerCamelCase    :       Optional[bool] = None       ,							__lowerCamelCase    :       Optional[bool] = None       ,							__lowerCamelCase    :       bool = False       ,							__lowerCamelCase    :       bool = False       ,							__lowerCamelCase    :       bool = False       ,							__lowerCamelCase    :       bool = False       ,							__lowerCamelCase    :       bool = True       ,							__lowerCamelCase    :       Optional[Union[str, TensorType]] = None       ,							**__lowerCamelCase    :       Dict       ,							)     -> BatchEncoding:






         """simple docstring"""







         UpperCAmelCase      					=  self.tokenizer(
             text=__lowerCamelCase       ,							add_special_tokens=__lowerCamelCase       ,							padding=__lowerCamelCase       ,							truncation=__lowerCamelCase       ,							max_length=__lowerCamelCase       ,							stride=__lowerCamelCase       ,							pad_to_multiple_of=__lowerCamelCase       ,							return_token_type_ids=__lowerCamelCase       ,							return_attention_mask=__lowerCamelCase       ,							return_overflowing_tokens=__lowerCamelCase       ,							return_special_tokens_mask=__lowerCamelCase       ,							return_offsets_mapping=__lowerCamelCase       ,							return_length=__lowerCamelCase       ,							verbose=__lowerCamelCase       ,							return_tensors=__lowerCamelCase       ,							**__lowerCamelCase       ,							)
         # add pixel_values + pixel_mask
         UpperCAmelCase      					=  self.image_processor(
             __lowerCamelCase       ,							return_tensors=__lowerCamelCase       ,							do_normalize=__lowerCamelCase       ,							do_center_crop=__lowerCamelCase       ,							**__lowerCamelCase							)
         encoding.update(__lowerCamelCase							)

         return encoding





      def 							_lowercase					(		self    :       Optional[Any]       ,							*__lowerCamelCase    :       Optional[int]       ,							**__lowerCamelCase    :       List[str]							)     -> str:






         """simple docstring"""







         return self.tokenizer.batch_decode(*__lowerCamelCase       ,							**__lowerCamelCase							)





      def 							_lowercase					(		self    :       Dict       ,							*__lowerCamelCase    :       Any       ,							**__lowerCamelCase    :       List[Any]							)     -> Optional[int]:






         """simple docstring"""







         return self.tokenizer.decode(*__lowerCamelCase       ,							**__lowerCamelCase							)





      @property
      def 							_lowercase					(		self    :       List[Any]							)     -> Optional[int]:






         """simple docstring"""







         UpperCAmelCase      					=  self.tokenizer.model_input_names
         UpperCAmelCase      					=  self.image_processor.model_input_names
         return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)



