





import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class   A      (lowercase__       ,					unittest.TestCase   ):
			'''simple docstring'''




			__lowerCamelCase     :	Any          =   BertJapaneseTokenizer
			__lowerCamelCase     :	int          =   False
			__lowerCamelCase     :	List[str]          =   True






			def        a_	(      self       :   Optional[Any]       )     ->     Dict:







					"""simple docstring"""






					super().setUp()

					A__  				=       [
					    """[UNK]""",
					    """[CLS]""",
					    """[SEP]""",
					    """こんにちは""",
					    """こん""",
					    """にちは""",
					    """ばんは""",
					    """##こん""",
					    """##にちは""",
					    """##ばんは""",
					    """世界""",
					    """##世界""",
					    """、""",
					    """##、""",
					    """。""",
					    """##。""",
					]

					A__  				=       os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["""vocab_file"""]       )
					with open(self.vocab_file							,     """w"""							,     encoding="""utf-8"""       ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






			def        a_	(      self       :   Dict							,     __lowerCAmelCase       :   Optional[Any]       )     ->     List[str]:







					"""simple docstring"""






					A__  				=       """こんにちは、世界。 \nこんばんは、世界。"""
					A__  				=       """こんにちは 、 世界 。 こんばんは 、 世界 。"""
					return input_text, output_text






			def        a_	(      self       :   List[Any]							,     __lowerCAmelCase       :   Optional[int]       )     ->     Dict:







					"""simple docstring"""






					A__  				=       self.get_input_output_texts(_UpperCamelCase       )
					A__  				=       tokenizer.encode(_UpperCamelCase							,     add_special_tokens=_UpperCamelCase       )
					A__  				=       tokenizer.decode(_UpperCamelCase							,     clean_up_tokenization_spaces=_UpperCamelCase       )
					return text, ids






			def        a_	(      self       :   Tuple       )     ->     int:







					"""simple docstring"""






					pass  # TODO add if relevant






			def        a_	(      self       :   Optional[Any]       )     ->     Union[str, Any]:







					"""simple docstring"""






					pass  # TODO add if relevant






			def        a_	(      self       :   Dict       )     ->     Union[str, Any]:







					"""simple docstring"""






					pass  # TODO add if relevant






			def        a_	(      self       :   List[str]       )     ->     Optional[int]:







					"""simple docstring"""






					A__  				=       self.tokenizer_class(self.vocab_file       )

					A__  				=       tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""       )
					self.assertListEqual(_UpperCamelCase							,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase       )							,     [3, 12, 10, 14, 4, 9, 12, 10, 14]       )






			def        a_	(      self       :   int       )     ->     List[str]:







					"""simple docstring"""






					A__  				=       self.tokenizer_class(self.vocab_file							,     word_tokenizer_type="""mecab"""       )
					self.assertIsNotNone(_UpperCamelCase       )

					A__  				=       """こんにちは、世界。\nこんばんは、世界。"""
					A__  				=       tokenizer.tokenize(_UpperCamelCase       )
					self.assertListEqual(_UpperCamelCase							,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase       )							,     [3, 12, 10, 14, 4, 9, 12, 10, 14]       )

					A__  				=       os.path.join(self.tmpdirname							,     """tokenizer.bin"""       )
					with open(_UpperCamelCase							,     """wb"""       ) as handle:
							pickle.dump(_UpperCamelCase							,     _UpperCamelCase       )

					with open(_UpperCamelCase							,     """rb"""       ) as handle:
							A__  				=       pickle.load(_UpperCamelCase       )

					A__  				=       tokenizer_new.tokenize(_UpperCamelCase       )

					self.assertListEqual(_UpperCamelCase							,     _UpperCamelCase       )






			def        a_	(      self       :   int       )     ->     Dict:







					"""simple docstring"""






					A__  				=       MecabTokenizer(mecab_dic="""ipadic"""       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,     )






			def        a_	(      self       :   Optional[Any]       )     ->     Any:







					"""simple docstring"""






					try:
							A__  				=       MecabTokenizer(mecab_dic="""unidic_lite"""       )
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,     )






			def        a_	(      self       :   Union[str, Any]       )     ->     Tuple:







					"""simple docstring"""






					try:
							A__  				=       MecabTokenizer(mecab_dic="""unidic"""       )
					except ModuleNotFoundError:
							return

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,     )






			def        a_	(      self       :   Optional[int]       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       MecabTokenizer(do_lower_case=_UpperCamelCase							,     mecab_dic="""ipadic"""       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,     )






			def        a_	(      self       :   List[str]       )     ->     Any:







					"""simple docstring"""






					try:
							A__  				=       MecabTokenizer(
							    do_lower_case=_UpperCamelCase							,     normalize_text=_UpperCamelCase							,     mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""       )
					except RuntimeError:
							# if dict doesn't exist in the system, previous code raises this error.
							return

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,     )






			def        a_	(      self       :   int       )     ->     Any:







					"""simple docstring"""






					A__  				=       MecabTokenizer(normalize_text=_UpperCamelCase							,     mecab_dic="""ipadic"""       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]							,     )






			@require_sudachi
			def        a_	(      self       :   Union[str, Any]       )     ->     int:







					"""simple docstring"""






					A__  				=       self.tokenizer_class(self.vocab_file							,     word_tokenizer_type="""sudachi"""       )
					self.assertIsNotNone(_UpperCamelCase       )

					A__  				=       """こんにちは、世界。\nこんばんは、世界。"""
					A__  				=       tokenizer.tokenize(_UpperCamelCase       )
					self.assertListEqual(_UpperCamelCase							,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase       )							,     [3, 12, 10, 14, 4, 9, 12, 10, 14]       )

					A__  				=       os.path.join(self.tmpdirname							,     """tokenizer.bin"""       )
					with open(_UpperCamelCase							,     """wb"""       ) as handle:
							pickle.dump(_UpperCamelCase							,     _UpperCamelCase       )

					with open(_UpperCamelCase							,     """rb"""       ) as handle:
							A__  				=       pickle.load(_UpperCamelCase       )

					A__  				=       tokenizer_new.tokenize(_UpperCamelCase       )

					self.assertListEqual(_UpperCamelCase							,     _UpperCamelCase       )






			@require_sudachi
			def        a_	(      self       :   str       )     ->     Any:







					"""simple docstring"""






					A__  				=       SudachiTokenizer(sudachi_dict_type="""core"""       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     [""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]							,     )






			@require_sudachi
			def        a_	(      self       :   Dict       )     ->     Dict:







					"""simple docstring"""






					A__  				=       SudachiTokenizer(sudachi_dict_type="""core"""							,     sudachi_split_mode="""A"""       )

					self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )							,     ["""外国""", """人""", """参政""", """権"""]       )






			@require_sudachi
			def        a_	(      self       :   List[Any]       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       SudachiTokenizer(sudachi_dict_type="""core"""							,     sudachi_split_mode="""B"""       )

					self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )							,     ["""外国人""", """参政権"""]       )






			@require_sudachi
			def        a_	(      self       :   List[str]       )     ->     Tuple:







					"""simple docstring"""






					A__  				=       SudachiTokenizer(sudachi_dict_type="""core"""							,     sudachi_split_mode="""C"""       )

					self.assertListEqual(tokenizer.tokenize("""外国人参政権"""       )							,     ["""外国人参政権"""]       )






			@require_sudachi
			def        a_	(      self       :   str       )     ->     str:







					"""simple docstring"""






					A__  				=       SudachiTokenizer(do_lower_case=_UpperCamelCase							,     sudachi_dict_type="""core"""       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     [""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]							,     )






			@require_sudachi
			def        a_	(      self       :   Any       )     ->     Optional[int]:







					"""simple docstring"""






					A__  				=       SudachiTokenizer(normalize_text=_UpperCamelCase							,     sudachi_dict_type="""core"""       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     [""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]							,     )






			@require_sudachi
			def        a_	(      self       :   List[Any]       )     ->     Dict:







					"""simple docstring"""






					A__  				=       SudachiTokenizer(trim_whitespace=_UpperCamelCase							,     sudachi_dict_type="""core"""       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,     )






			@require_jumanpp
			def        a_	(      self       :   Tuple       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       self.tokenizer_class(self.vocab_file							,     word_tokenizer_type="""jumanpp"""       )
					self.assertIsNotNone(_UpperCamelCase       )

					A__  				=       """こんにちは、世界。\nこんばんは、世界。"""
					A__  				=       tokenizer.tokenize(_UpperCamelCase       )
					self.assertListEqual(_UpperCamelCase							,     ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]       )
					self.assertListEqual(tokenizer.convert_tokens_to_ids(_UpperCamelCase       )							,     [3, 12, 10, 14, 4, 9, 12, 10, 14]       )

					A__  				=       os.path.join(self.tmpdirname							,     """tokenizer.bin"""       )
					with open(_UpperCamelCase							,     """wb"""       ) as handle:
							pickle.dump(_UpperCamelCase							,     _UpperCamelCase       )

					with open(_UpperCamelCase							,     """rb"""       ) as handle:
							A__  				=       pickle.load(_UpperCamelCase       )

					A__  				=       tokenizer_new.tokenize(_UpperCamelCase       )

					self.assertListEqual(_UpperCamelCase							,     _UpperCamelCase       )






			@require_jumanpp
			def        a_	(      self       :   Any       )     ->     Any:







					"""simple docstring"""






					A__  				=       JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,     )






			@require_jumanpp
			def        a_	(      self       :   Optional[Any]       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       JumanppTokenizer(do_lower_case=_UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,     )






			@require_jumanpp
			def        a_	(      self       :   List[Any]       )     ->     List[Any]:







					"""simple docstring"""






					A__  				=       JumanppTokenizer(normalize_text=_UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,     )






			@require_jumanpp
			def        a_	(      self       :   Tuple       )     ->     Any:







					"""simple docstring"""






					A__  				=       JumanppTokenizer(trim_whitespace=_UpperCamelCase       )

					self.assertListEqual(
					    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """       )							,     ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]							,     )






			@require_jumanpp
			def        a_	(      self       :   Any       )     ->     Any:







					"""simple docstring"""






					A__  				=       JumanppTokenizer()

					self.assertListEqual(
					    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""       )							,     ["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]							,     )






			def        a_	(      self       :   Dict       )     ->     int:







					"""simple docstring"""






					A__  				=       ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

					A__  				=       {}
					for i, token in enumerate(_UpperCamelCase       ):
							A__  				=       i
					A__  				=       WordpieceTokenizer(vocab=_UpperCamelCase							,     unk_token="""[UNK]"""       )

					self.assertListEqual(tokenizer.tokenize(""""""       )							,     []       )

					self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )							,     ["""こんにちは"""]       )

					self.assertListEqual(tokenizer.tokenize("""こんばんは"""       )							,     ["""こん""", """##ばんは"""]       )

					self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""       )							,     ["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]       )






			def        a_	(      self       :   Any       )     ->     Union[str, Any]:







					"""simple docstring"""






					A__  				=       BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""       )
					A__  				=       tokenizer.subword_tokenizer

					A__  				=       subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""       )
					self.assertListEqual(_UpperCamelCase							,     ["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]       )

					A__  				=       subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""       )
					self.assertListEqual(_UpperCamelCase							,     ["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]       )







			def        a_	(      self       :   Optional[int]       )     ->     int:







					"""simple docstring"""






					A__  				=       self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""       )

					A__  				=       tokenizer.encode("""ありがとう。"""							,     add_special_tokens=_UpperCamelCase       )
					A__  				=       tokenizer.encode("""どういたしまして。"""							,     add_special_tokens=_UpperCamelCase       )

					A__  				=       tokenizer.build_inputs_with_special_tokens(_UpperCamelCase       )
					A__  				=       tokenizer.build_inputs_with_special_tokens(_UpperCamelCase							,     _UpperCamelCase       )

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class   A      (lowercase__       ,					unittest.TestCase   ):
			'''simple docstring'''




			__lowerCamelCase     :	Any          =   BertJapaneseTokenizer
			__lowerCamelCase     :	Dict          =   False






			def        a_	(      self       :   List[Any]       )     ->     Any:







					"""simple docstring"""






					super().setUp()

					A__  				=       ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

					A__  				=       os.path.join(self.tmpdirname							,     VOCAB_FILES_NAMES["""vocab_file"""]       )
					with open(self.vocab_file							,     """w"""							,     encoding="""utf-8"""       ) as vocab_writer:
							vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]       )       )






			def        a_	(      self       :   List[Any]							,     **__lowerCAmelCase       :   Optional[int]       )     ->     Optional[Any]:







					"""simple docstring"""






					return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,     subword_tokenizer_type="""character"""							,     **_UpperCamelCase       )






			def        a_	(      self       :   Union[str, Any]							,     __lowerCAmelCase       :   int       )     ->     Optional[int]:







					"""simple docstring"""






					A__  				=       """こんにちは、世界。 \nこんばんは、世界。"""
					A__  				=       """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
					return input_text, output_text






			def        a_	(      self       :   Optional[int]       )     ->     Dict:







					"""simple docstring"""






					pass  # TODO add if relevant






			def        a_	(      self       :   Union[str, Any]       )     ->     Optional[int]:







					"""simple docstring"""






					pass  # TODO add if relevant






			def        a_	(      self       :   Optional[Any]       )     ->     int:







					"""simple docstring"""






					pass  # TODO add if relevant






			def        a_	(      self       :   Optional[Any]       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       self.tokenizer_class(self.vocab_file							,     subword_tokenizer_type="""character"""       )

					A__  				=       tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""       )
					self.assertListEqual(
					    _UpperCamelCase							,     ["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]       )
					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(_UpperCamelCase       )							,     [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]       )






			def        a_	(      self       :   List[str]       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

					A__  				=       {}
					for i, token in enumerate(_UpperCamelCase       ):
							A__  				=       i
					A__  				=       CharacterTokenizer(vocab=_UpperCamelCase							,     unk_token="""[UNK]"""       )

					self.assertListEqual(tokenizer.tokenize(""""""       )							,     []       )

					self.assertListEqual(tokenizer.tokenize("""こんにちは"""       )							,     ["""こ""", """ん""", """に""", """ち""", """は"""]       )

					self.assertListEqual(tokenizer.tokenize("""こんにちほ"""       )							,     ["""こ""", """ん""", """に""", """ち""", """[UNK]"""]       )







			def        a_	(      self       :   Any       )     ->     str:







					"""simple docstring"""






					A__  				=       self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""       )

					A__  				=       tokenizer.encode("""ありがとう。"""							,     add_special_tokens=_UpperCamelCase       )
					A__  				=       tokenizer.encode("""どういたしまして。"""							,     add_special_tokens=_UpperCamelCase       )

					A__  				=       tokenizer.build_inputs_with_special_tokens(_UpperCamelCase       )
					A__  				=       tokenizer.build_inputs_with_special_tokens(_UpperCamelCase							,     _UpperCamelCase       )

					# 2 is for "[CLS]", 3 is for "[SEP]"
					assert encoded_sentence == [2] + text + [3]
					assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class   A      (unittest.TestCase   ):
			'''simple docstring'''




			def        a_	(      self       :   Dict       )     ->     Optional[int]:







					"""simple docstring"""






					A__  				=       """cl-tohoku/bert-base-japanese"""
					A__  				=       AutoTokenizer.from_pretrained(_UpperCamelCase       )
					self.assertIsInstance(_UpperCamelCase							,     _UpperCamelCase       )


class   A      (unittest.TestCase   ):
			'''simple docstring'''




			def        a_	(      self       :   Any       )     ->     int:







					"""simple docstring"""






					A__  				=       """cl-tohoku/bert-base-japanese"""
					with self.assertLogs("""transformers"""							,     level="""WARNING"""       ) as cm:
							BertTokenizer.from_pretrained(_UpperCamelCase       )
							self.assertTrue(
							    cm.records[0].message.startswith(
							        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
							        """ is called from."""       )       )
					A__  				=       """bert-base-cased"""
					with self.assertLogs("""transformers"""							,     level="""WARNING"""       ) as cm:
							BertJapaneseTokenizer.from_pretrained(_UpperCamelCase       )
							self.assertTrue(
							    cm.records[0].message.startswith(
							        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
							        """ is called from."""       )       )










from __future__ import annotations

import unittest

from transformers import BlenderbotSmallConfig, BlenderbotSmallTokenizer, is_tf_available
from transformers.testing_utils import require_tf, require_tokenizers, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
					import tensorflow as tf

					from transformers import TFAutoModelForSeqaSeqLM, TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel


@require_tf
class   A      :
			'''simple docstring'''




			__lowerCamelCase     :	Optional[Any]          =   BlenderbotSmallConfig
			__lowerCamelCase     :	Optional[Any]          =   {}
			__lowerCamelCase     :	List[Any]          =   '''gelu'''






			def __init__(      self       :   Dict							,     __lowerCAmelCase       :   Tuple							,     __lowerCAmelCase       :   List[str]=13							,     __lowerCAmelCase       :   List[Any]=7							,     __lowerCAmelCase       :   List[str]=True							,     __lowerCAmelCase       :   List[Any]=False							,     __lowerCAmelCase       :   Union[str, Any]=99							,     __lowerCAmelCase       :   Union[str, Any]=32							,     __lowerCAmelCase       :   Any=2							,     __lowerCAmelCase       :   Optional[Any]=4							,     __lowerCAmelCase       :   Tuple=37							,     __lowerCAmelCase       :   List[Any]=0.1							,     __lowerCAmelCase       :   Optional[int]=0.1							,     __lowerCAmelCase       :   List[str]=20							,     __lowerCAmelCase       :   Union[str, Any]=2							,     __lowerCAmelCase       :   Dict=1							,     __lowerCAmelCase       :   int=0							,     )     ->     Any:







					"""simple docstring"""






					A__  				=       parent
					A__  				=       batch_size
					A__  				=       seq_length
					A__  				=       is_training
					A__  				=       use_labels
					A__  				=       vocab_size
					A__  				=       hidden_size
					A__  				=       num_hidden_layers
					A__  				=       num_attention_heads
					A__  				=       intermediate_size

					A__  				=       hidden_dropout_prob
					A__  				=       attention_probs_dropout_prob
					A__  				=       max_position_embeddings
					A__  				=       eos_token_id
					A__  				=       pad_token_id
					A__  				=       bos_token_id






			def        a_	(      self       :   Optional[Any]       )     ->     Tuple:







					"""simple docstring"""






					A__  				=       ids_tensor([self.batch_size, self.seq_length - 1]							,     self.vocab_size       )
					A__  				=       tf.expand_dims(tf.constant([self.eos_token_id] * self.batch_size       )							,     1       )
					A__  				=       tf.concat([input_ids, eos_tensor]							,     axis=1       )

					A__  				=       ids_tensor([self.batch_size, self.seq_length]							,     self.vocab_size       )

					A__  				=       self.config_cls(
					    vocab_size=self.vocab_size							,     d_model=self.hidden_size							,     encoder_layers=self.num_hidden_layers							,     decoder_layers=self.num_hidden_layers							,     encoder_attention_heads=self.num_attention_heads							,     decoder_attention_heads=self.num_attention_heads							,     encoder_ffn_dim=self.intermediate_size							,     decoder_ffn_dim=self.intermediate_size							,     dropout=self.hidden_dropout_prob							,     attention_dropout=self.attention_probs_dropout_prob							,     max_position_embeddings=self.max_position_embeddings							,     eos_token_ids=[2]							,     bos_token_id=self.bos_token_id							,     pad_token_id=self.pad_token_id							,     decoder_start_token_id=self.pad_token_id							,     **self.config_updates							,     )
					A__  				=       prepare_blenderbot_small_inputs_dict(__lowerCAmelCase							,     __lowerCAmelCase							,     __lowerCAmelCase       )
					return config, inputs_dict







			def        a_	(      self       :   Union[str, Any]							,     __lowerCAmelCase       :   Any							,     __lowerCAmelCase       :   Union[str, Any]       )     ->     str:







					"""simple docstring"""






					A__  				=       TFBlenderbotSmallModel(config=__lowerCAmelCase       ).get_decoder()
					A__  				=       inputs_dict["""input_ids"""]

					A__  				=       input_ids[:1, :]
					A__  				=       inputs_dict["""attention_mask"""][:1, :]
					A__  				=       inputs_dict["""head_mask"""]
					A__  				=       1

					# first forward pass
					A__  				=       model(__lowerCAmelCase							,     attention_mask=__lowerCAmelCase							,     head_mask=__lowerCAmelCase							,     use_cache=__lowerCAmelCase       )

					A__				,      A__  				=       outputs.to_tuple()

					# create hypothetical next token and extent to next_input_ids
					A__  				=       ids_tensor((self.batch_size, 3)							,     config.vocab_size       )
					A__  				=       tf.cast(ids_tensor((self.batch_size, 3)							,     2       )							,     tf.inta       )

					# append to next input_ids and
					A__  				=       tf.concat([input_ids, next_tokens]							,     axis=-1       )
					A__  				=       tf.concat([attention_mask, next_attn_mask]							,     axis=-1       )

					A__  				=       model(__lowerCAmelCase							,     attention_mask=__lowerCAmelCase       )[0]
					A__  				=       model(__lowerCAmelCase							,     attention_mask=__lowerCAmelCase							,     past_key_values=__lowerCAmelCase       )[0]

					self.parent.assertEqual(next_tokens.shape[1]							,     output_from_past.shape[1]       )

					# select random slice
					A__  				=       int(ids_tensor((1,)							,     output_from_past.shape[-1]       )       )
					A__  				=       output_from_no_past[:, -3:, random_slice_idx]
					A__  				=       output_from_past[:, :, random_slice_idx]

					# test that outputs are equal for slice
					tf.debugging.assert_near(__lowerCAmelCase							,     __lowerCAmelCase							,     rtol=1e-3       )







def        __lowerCamelCase (							__a       :Dict   ,					__a       :Tuple   ,					__a       :List[Any]   ,					__a       :List[str]=None   ,					__a       :List[Any]=None   ,					__a       :Optional[Any]=None   ,					__a       :List[str]=None   ,					__a       :int=None   ,					)		->    Optional[Any]:






		"""simple docstring"""

		if attention_mask is None:
				A__  				=       tf.cast(tf.math.not_equal(__a   ,					config.pad_token_id   )   ,					tf.inta   )
		if decoder_attention_mask is None:
				A__  				=       tf.concat(
				    [
				        tf.ones(decoder_input_ids[:, :1].shape   ,					dtype=tf.inta   ),
				        tf.cast(tf.math.not_equal(decoder_input_ids[:, 1:]   ,					config.pad_token_id   )   ,					tf.inta   ),
				    ]   ,					axis=-1   ,					)
		if head_mask is None:
				A__  				=       tf.ones((config.encoder_layers, config.encoder_attention_heads)   )
		if decoder_head_mask is None:
				A__  				=       tf.ones((config.decoder_layers, config.decoder_attention_heads)   )
		if cross_attn_head_mask is None:
				A__  				=       tf.ones((config.decoder_layers, config.decoder_attention_heads)   )
		return {
		    "input_ids": input_ids,
		    "decoder_input_ids": decoder_input_ids,
		    "attention_mask": attention_mask,
		    "decoder_attention_mask": decoder_attention_mask,
		    "head_mask": head_mask,
		    "decoder_head_mask": decoder_head_mask,
		    "cross_attn_head_mask": cross_attn_head_mask,
		}


@require_tf
class   A      (SCREAMING_SNAKE_CASE       ,					SCREAMING_SNAKE_CASE       ,					unittest.TestCase   ):
			'''simple docstring'''




			__lowerCamelCase     :	Tuple          =   (
			    (TFBlenderbotSmallForConditionalGeneration, TFBlenderbotSmallModel) if is_tf_available() else ()
			)
			__lowerCamelCase     :	List[Any]          =   (TFBlenderbotSmallForConditionalGeneration,) if is_tf_available() else ()
			__lowerCamelCase     :	Tuple          =   (
			    {
			        '''conversational''': TFBlenderbotSmallForConditionalGeneration,
			        '''feature-extraction''': TFBlenderbotSmallModel,
			        '''summarization''': TFBlenderbotSmallForConditionalGeneration,
			        '''text2text-generation''': TFBlenderbotSmallForConditionalGeneration,
			        '''translation''': TFBlenderbotSmallForConditionalGeneration,
			    }
			    if is_tf_available()
			    else {}
			)
			__lowerCamelCase     :	Dict          =   True
			__lowerCamelCase     :	Optional[Any]          =   False
			__lowerCamelCase     :	Tuple          =   False






			def        a_	(      self       :   Tuple       )     ->     Optional[int]:







					"""simple docstring"""






					A__  				=       TFBlenderbotSmallModelTester(self       )
					A__  				=       ConfigTester(self							,     config_class=__lowerCAmelCase       )






			def        a_	(      self       :   List[str]       )     ->     int:







					"""simple docstring"""






					self.config_tester.run_common_tests()







			def        a_	(      self       :   List[str]       )     ->     Any:







					"""simple docstring"""






					A__  				=       self.model_tester.prepare_config_and_inputs_for_common()
					self.model_tester.check_decoder_model_past_large_inputs(*__lowerCAmelCase       )


@require_tokenizers
@require_tf
class   A      (unittest.TestCase   ):
			'''simple docstring'''




			__lowerCamelCase     :	List[str]          =   [
			    '''Social anxiety\nWow, I am never shy. Do you have anxiety?\nYes. I end up sweating and blushing and feel like '''
			    '''  i\'m going to throw up.\nand why is that?'''
			]
			__lowerCamelCase     :	Optional[int]          =   '''facebook/blenderbot_small-90M'''






			@cached_property
			def        a_	(      self       :   Optional[int]       )     ->     List[str]:







					"""simple docstring"""






					return BlenderbotSmallTokenizer.from_pretrained("""facebook/blenderbot-90M"""       )






			@cached_property
			def        a_	(      self       :   List[str]       )     ->     List[str]:







					"""simple docstring"""






					A__  				=       TFAutoModelForSeqaSeqLM.from_pretrained(self.model_name       )
					return model







			@slow
			def        a_	(      self       :   int       )     ->     Optional[Any]:







					"""simple docstring"""






					A__  				=       self.tokenizer(self.src_text							,     return_tensors="""tf"""       )
					A__  				=       self.model.generate(
					    model_inputs.input_ids							,     attention_mask=model_inputs.attention_mask							,     num_beams=2							,     use_cache=__lowerCAmelCase							,     )
					A__  				=       self.tokenizer.batch_decode(generated_ids.numpy()							,     skip_special_tokens=__lowerCAmelCase       )[0]
					assert generated_words in (
					    "i don't know. i just feel like i'm going to throw up. it's not fun.",
					    "i'm not sure. i just feel like i've been feeling like i have to be in a certain place",
					    "i'm not sure. i just feel like i've been in a bad situation.",
					)



