import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class        __snake_case  (__UpperCAmelCase						,	unittest.TestCase		):
				lowerCAmelCase__				=     BertJapaneseTokenizer
				lowerCAmelCase__				=     False
				lowerCAmelCase__				=     True



				def       SCREAMING_SNAKE_CASE       (      self      :		Optional[Any]      )  ->    str:





								'''simple docstring'''




								super().setUp()

								_lowerCAmelCase    :    List[str]										=      [
								    """[UNK]""",
								    """[CLS]""",
								    """[SEP]""",
								    """こんにちは""",
								    """こん""",
								    """にちは""",
								    """ばんは""",
								    """##こん""",
								    """##にちは""",
								    """##ばんは""",
								    """世界""",
								    """##世界""",
								    """、""",
								    """##、""",
								    """。""",
								    """##。""",
								]

								_lowerCAmelCase    :    Union[str, Any]										=      os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES["""vocab_file"""]      )
								with open(self.vocab_file		,	"""w"""		,	encoding="""utf-8"""      ) as vocab_writer:
												vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Union[str, Any]		,	_UpperCAmelCase      :		Optional[Any]      )  ->    Tuple:





								'''simple docstring'''




								_lowerCAmelCase    :    str										=      """こんにちは、世界。 \nこんばんは、世界。"""
								_lowerCAmelCase    :    Tuple										=      """こんにちは 、 世界 。 こんばんは 、 世界 。"""
								return input_text, output_text



				def       SCREAMING_SNAKE_CASE       (      self      :		str		,	_UpperCAmelCase      :		List[str]      )  ->    Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase					,    _lowerCAmelCase    :    str										=      self.get_input_output_texts(UpperCAmelCase_      )
								_lowerCAmelCase    :    str										=      tokenizer.encode(UpperCAmelCase_		,	add_special_tokens=UpperCAmelCase_      )
								_lowerCAmelCase    :    List[str]										=      tokenizer.decode(UpperCAmelCase_		,	clean_up_tokenization_spaces=UpperCAmelCase_      )
								return text, ids



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    Any:





								'''simple docstring'''




								pass  # TODO add if relevant



				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    List[str]:





								'''simple docstring'''




								pass  # TODO add if relevant



				def       SCREAMING_SNAKE_CASE       (      self      :		int      )  ->    Optional[Any]:





								'''simple docstring'''




								pass  # TODO add if relevant



				def       SCREAMING_SNAKE_CASE       (      self      :		Optional[Any]      )  ->    int:





								'''simple docstring'''




								_lowerCAmelCase    :    Dict										=      self.tokenizer_class(self.vocab_file      )

								_lowerCAmelCase    :    Any										=      tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""      )
								self.assertListEqual(UpperCAmelCase_		,	["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )		,	[3, 12, 10, 14, 4, 9, 12, 10, 14]      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    Tuple:





								'''simple docstring'''




								_lowerCAmelCase    :    Dict										=      self.tokenizer_class(self.vocab_file		,	word_tokenizer_type="""mecab"""      )
								self.assertIsNotNone(UpperCAmelCase_      )

								_lowerCAmelCase    :    Dict										=      """こんにちは、世界。\nこんばんは、世界。"""
								_lowerCAmelCase    :    Tuple										=      tokenizer.tokenize(UpperCAmelCase_      )
								self.assertListEqual(UpperCAmelCase_		,	["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )		,	[3, 12, 10, 14, 4, 9, 12, 10, 14]      )

								_lowerCAmelCase    :    int										=      os.path.join(self.tmpdirname		,	"""tokenizer.bin"""      )
								with open(UpperCAmelCase_		,	"""wb"""      ) as handle:
												pickle.dump(UpperCAmelCase_		,	UpperCAmelCase_      )

								with open(UpperCAmelCase_		,	"""rb"""      ) as handle:
												_lowerCAmelCase    :    List[str]										=      pickle.load(UpperCAmelCase_      )

								_lowerCAmelCase    :    Union[str, Any]										=      tokenizer_new.tokenize(UpperCAmelCase_      )

								self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Optional[Any]      )  ->    Union[str, Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[str]										=      MecabTokenizer(mecab_dic="""ipadic"""      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,	)



				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    Union[str, Any]:





								'''simple docstring'''




								try:
												_lowerCAmelCase    :    Dict										=      MecabTokenizer(mecab_dic="""unidic_lite"""      )
								except ModuleNotFoundError:
												return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,	)



				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    Dict:





								'''simple docstring'''




								try:
												_lowerCAmelCase    :    List[str]										=      MecabTokenizer(mecab_dic="""unidic"""      )
								except ModuleNotFoundError:
												return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,	)



				def       SCREAMING_SNAKE_CASE       (      self      :		Dict      )  ->    Tuple:





								'''simple docstring'''




								_lowerCAmelCase    :    Any										=      MecabTokenizer(do_lower_case=UpperCAmelCase_		,	mecab_dic="""ipadic"""      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,	)



				def       SCREAMING_SNAKE_CASE       (      self      :		int      )  ->    Tuple:





								'''simple docstring'''




								try:
												_lowerCAmelCase    :    Any										=      MecabTokenizer(
												    do_lower_case=UpperCAmelCase_		,	normalize_text=UpperCAmelCase_		,	mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""      )
								except RuntimeError:
												# if dict doesn't exist in the system, previous code raises this error.
												return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,	)



				def       SCREAMING_SNAKE_CASE       (      self      :		Any      )  ->    Tuple:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      MecabTokenizer(normalize_text=UpperCAmelCase_		,	mecab_dic="""ipadic"""      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]		,	)



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		Dict      )  ->    Any:





								'''simple docstring'''




								_lowerCAmelCase    :    Tuple										=      self.tokenizer_class(self.vocab_file		,	word_tokenizer_type="""sudachi"""      )
								self.assertIsNotNone(UpperCAmelCase_      )

								_lowerCAmelCase    :    Tuple										=      """こんにちは、世界。\nこんばんは、世界。"""
								_lowerCAmelCase    :    List[str]										=      tokenizer.tokenize(UpperCAmelCase_      )
								self.assertListEqual(UpperCAmelCase_		,	["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )		,	[3, 12, 10, 14, 4, 9, 12, 10, 14]      )

								_lowerCAmelCase    :    int										=      os.path.join(self.tmpdirname		,	"""tokenizer.bin"""      )
								with open(UpperCAmelCase_		,	"""wb"""      ) as handle:
												pickle.dump(UpperCAmelCase_		,	UpperCAmelCase_      )

								with open(UpperCAmelCase_		,	"""rb"""      ) as handle:
												_lowerCAmelCase    :    Union[str, Any]										=      pickle.load(UpperCAmelCase_      )

								_lowerCAmelCase    :    Optional[int]										=      tokenizer_new.tokenize(UpperCAmelCase_      )

								self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_      )



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		Optional[int]      )  ->    Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[Any]										=      SudachiTokenizer(sudachi_dict_type="""core"""      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]		,	)



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		Any      )  ->    List[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      SudachiTokenizer(sudachi_dict_type="""core"""		,	sudachi_split_mode="""A"""      )

								self.assertListEqual(tokenizer.tokenize("""外国人参政権"""      )		,	["""外国""", """人""", """参政""", """権"""]      )



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		Dict      )  ->    Dict:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      SudachiTokenizer(sudachi_dict_type="""core"""		,	sudachi_split_mode="""B"""      )

								self.assertListEqual(tokenizer.tokenize("""外国人参政権"""      )		,	["""外国人""", """参政権"""]      )



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    Tuple:





								'''simple docstring'''




								_lowerCAmelCase    :    Tuple										=      SudachiTokenizer(sudachi_dict_type="""core"""		,	sudachi_split_mode="""C"""      )

								self.assertListEqual(tokenizer.tokenize("""外国人参政権"""      )		,	["""外国人参政権"""]      )



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    Dict:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      SudachiTokenizer(do_lower_case=UpperCAmelCase_		,	sudachi_dict_type="""core"""      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]		,	)



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    List[str]:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[Any]										=      SudachiTokenizer(normalize_text=UpperCAmelCase_		,	sudachi_dict_type="""core"""      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]		,	)



				@require_sudachi
				def       SCREAMING_SNAKE_CASE       (      self      :		str      )  ->    Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    int										=      SudachiTokenizer(trim_whitespace=UpperCAmelCase_		,	sudachi_dict_type="""core"""      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]		,	)



				@require_jumanpp
				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      self.tokenizer_class(self.vocab_file		,	word_tokenizer_type="""jumanpp"""      )
								self.assertIsNotNone(UpperCAmelCase_      )

								_lowerCAmelCase    :    List[str]										=      """こんにちは、世界。\nこんばんは、世界。"""
								_lowerCAmelCase    :    str										=      tokenizer.tokenize(UpperCAmelCase_      )
								self.assertListEqual(UpperCAmelCase_		,	["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]      )
								self.assertListEqual(tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )		,	[3, 12, 10, 14, 4, 9, 12, 10, 14]      )

								_lowerCAmelCase    :    Union[str, Any]										=      os.path.join(self.tmpdirname		,	"""tokenizer.bin"""      )
								with open(UpperCAmelCase_		,	"""wb"""      ) as handle:
												pickle.dump(UpperCAmelCase_		,	UpperCAmelCase_      )

								with open(UpperCAmelCase_		,	"""rb"""      ) as handle:
												_lowerCAmelCase    :    Tuple										=      pickle.load(UpperCAmelCase_      )

								_lowerCAmelCase    :    Dict										=      tokenizer_new.tokenize(UpperCAmelCase_      )

								self.assertListEqual(UpperCAmelCase_		,	UpperCAmelCase_      )



				@require_jumanpp
				def       SCREAMING_SNAKE_CASE       (      self      :		str      )  ->    Union[str, Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[Any]										=      JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,	)



				@require_jumanpp
				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    Union[str, Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Tuple										=      JumanppTokenizer(do_lower_case=UpperCAmelCase_      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,	)



				@require_jumanpp
				def       SCREAMING_SNAKE_CASE       (      self      :		Union[str, Any]      )  ->    Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[str]										=      JumanppTokenizer(normalize_text=UpperCAmelCase_      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]		,	)



				@require_jumanpp
				def       SCREAMING_SNAKE_CASE       (      self      :		int      )  ->    Optional[int]:





								'''simple docstring'''




								_lowerCAmelCase    :    int										=      JumanppTokenizer(trim_whitespace=UpperCAmelCase_      )

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """      )		,	["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]		,	)



				@require_jumanpp
				def       SCREAMING_SNAKE_CASE       (      self      :		List[Any]      )  ->    Union[str, Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Union[str, Any]										=      JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""      )		,	["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]		,	)



				def       SCREAMING_SNAKE_CASE       (      self      :		Optional[int]      )  ->    Tuple:





								'''simple docstring'''




								_lowerCAmelCase    :    List[str]										=      ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

								_lowerCAmelCase    :    List[str]										=      {}
								for i, token in enumerate(UpperCAmelCase_      ):
												_lowerCAmelCase    :    Optional[int]										=      i
								_lowerCAmelCase    :    Tuple										=      WordpieceTokenizer(vocab=UpperCAmelCase_		,	unk_token="""[UNK]"""      )

								self.assertListEqual(tokenizer.tokenize(""""""      )		,	[]      )

								self.assertListEqual(tokenizer.tokenize("""こんにちは"""      )		,	["""こんにちは"""]      )

								self.assertListEqual(tokenizer.tokenize("""こんばんは"""      )		,	["""こん""", """##ばんは"""]      )

								self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""      )		,	["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]      )



				def       SCREAMING_SNAKE_CASE       (      self      :		int      )  ->    str:





								'''simple docstring'''




								_lowerCAmelCase    :    int										=      BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""      )
								_lowerCAmelCase    :    Tuple										=      tokenizer.subword_tokenizer

								_lowerCAmelCase    :    Optional[Any]										=      subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""      )
								self.assertListEqual(UpperCAmelCase_		,	["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]      )

								_lowerCAmelCase    :    Optional[int]										=      subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""      )
								self.assertListEqual(UpperCAmelCase_		,	["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    Dict:





								'''simple docstring'''




								_lowerCAmelCase    :    Tuple										=      self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""      )

								_lowerCAmelCase    :    Optional[int]										=      tokenizer.encode("""ありがとう。"""		,	add_special_tokens=UpperCAmelCase_      )
								_lowerCAmelCase    :    Optional[Any]										=      tokenizer.encode("""どういたしまして。"""		,	add_special_tokens=UpperCAmelCase_      )

								_lowerCAmelCase    :    Any										=      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_      )
								_lowerCAmelCase    :    Optional[Any]										=      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_		,	UpperCAmelCase_      )

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class        __snake_case  (__UpperCAmelCase						,	unittest.TestCase		):
				lowerCAmelCase__				=     BertJapaneseTokenizer
				lowerCAmelCase__				=     False



				def       SCREAMING_SNAKE_CASE       (      self      :		str      )  ->    Union[str, Any]:





								'''simple docstring'''




								super().setUp()

								_lowerCAmelCase    :    Union[str, Any]										=      ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

								_lowerCAmelCase    :    List[str]										=      os.path.join(self.tmpdirname		,	VOCAB_FILES_NAMES["""vocab_file"""]      )
								with open(self.vocab_file		,	"""w"""		,	encoding="""utf-8"""      ) as vocab_writer:
												vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]      )      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Any		,	**_UpperCAmelCase      :		Optional[int]      )  ->    List[Any]:





								'''simple docstring'''




								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname		,	subword_tokenizer_type="""character"""		,	**UpperCAmelCase_      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Dict		,	_UpperCAmelCase      :		List[Any]      )  ->    List[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      """こんにちは、世界。 \nこんばんは、世界。"""
								_lowerCAmelCase    :    Dict										=      """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
								return input_text, output_text



				def       SCREAMING_SNAKE_CASE       (      self      :		List[Any]      )  ->    Dict:





								'''simple docstring'''




								pass  # TODO add if relevant



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    List[Any]:





								'''simple docstring'''




								pass  # TODO add if relevant



				def       SCREAMING_SNAKE_CASE       (      self      :		Tuple      )  ->    Optional[int]:





								'''simple docstring'''




								pass  # TODO add if relevant



				def       SCREAMING_SNAKE_CASE       (      self      :		Union[str, Any]      )  ->    List[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    List[str]										=      self.tokenizer_class(self.vocab_file		,	subword_tokenizer_type="""character"""      )

								_lowerCAmelCase    :    Any										=      tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""      )
								self.assertListEqual(
								    UpperCAmelCase_		,	["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]      )
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(UpperCAmelCase_      )		,	[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Dict      )  ->    List[str]:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

								_lowerCAmelCase    :    Optional[int]										=      {}
								for i, token in enumerate(UpperCAmelCase_      ):
												_lowerCAmelCase    :    Dict										=      i
								_lowerCAmelCase    :    Optional[int]										=      CharacterTokenizer(vocab=UpperCAmelCase_		,	unk_token="""[UNK]"""      )

								self.assertListEqual(tokenizer.tokenize(""""""      )		,	[]      )

								self.assertListEqual(tokenizer.tokenize("""こんにちは"""      )		,	["""こ""", """ん""", """に""", """ち""", """は"""]      )

								self.assertListEqual(tokenizer.tokenize("""こんにちほ"""      )		,	["""こ""", """ん""", """に""", """ち""", """[UNK]"""]      )



				def       SCREAMING_SNAKE_CASE       (      self      :		str      )  ->    Optional[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Tuple										=      self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""      )

								_lowerCAmelCase    :    Any										=      tokenizer.encode("""ありがとう。"""		,	add_special_tokens=UpperCAmelCase_      )
								_lowerCAmelCase    :    Any										=      tokenizer.encode("""どういたしまして。"""		,	add_special_tokens=UpperCAmelCase_      )

								_lowerCAmelCase    :    int										=      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_      )
								_lowerCAmelCase    :    Optional[Any]										=      tokenizer.build_inputs_with_special_tokens(UpperCAmelCase_		,	UpperCAmelCase_      )

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class        __snake_case  (unittest.TestCase		):



				def       SCREAMING_SNAKE_CASE       (      self      :		Dict      )  ->    List[str]:





								'''simple docstring'''




								_lowerCAmelCase    :    str										=      """cl-tohoku/bert-base-japanese"""
								_lowerCAmelCase    :    int										=      AutoTokenizer.from_pretrained(UpperCAmelCase_      )
								self.assertIsInstance(UpperCAmelCase_		,	UpperCAmelCase_      )






class        __snake_case  (unittest.TestCase		):



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    int:





								'''simple docstring'''




								_lowerCAmelCase    :    str										=      """cl-tohoku/bert-base-japanese"""
								with self.assertLogs("""transformers"""		,	level="""WARNING"""      ) as cm:
												BertTokenizer.from_pretrained(UpperCAmelCase_      )
												self.assertTrue(
												    cm.records[0].message.startswith(
												        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
												        """ is called from."""      )      )
								_lowerCAmelCase    :    Union[str, Any]										=      """bert-base-cased"""
								with self.assertLogs("""transformers"""		,	level="""WARNING"""      ) as cm:
												BertJapaneseTokenizer.from_pretrained(UpperCAmelCase_      )
												self.assertTrue(
												    cm.records[0].message.startswith(
												        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
												        """ is called from."""      )      )







from __future__ import annotations

from collections.abc import Iterator
from typing import Any
class        __snake_case  :



				def __init__(      self      :		List[Any]		,	_UpperCAmelCase      :		Any      )  ->    Dict:





								'''simple docstring'''




								_lowerCAmelCase    :    Any										=      data
								_lowerCAmelCase    :    Node | None										=      None






class        __snake_case  :



				def __init__(      self      :		Tuple      )  ->    Tuple:





								'''simple docstring'''




								_lowerCAmelCase    :    str										=      None
								_lowerCAmelCase    :    List[Any]										=      None



				def __iter__(      self      :		str      )  ->    Iterator[Any]:





								'''simple docstring'''




								_lowerCAmelCase    :    Optional[int]										=      self.head
								while self.head:
												yield node.data
												_lowerCAmelCase    :    List[str]										=      node.next
												if node == self.head:
																break



				def __len__(      self      :		Union[str, Any]      )  ->    int:





								'''simple docstring'''




								return sum(1 for _ in self      )



				def __repr__(      self      :		Optional[int]      )  ->    str:





								'''simple docstring'''




								return "->".join(str(_UpperCAmelCase      ) for item in iter(self      )      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Any		,	_UpperCAmelCase      :		Any      )  ->    None:





								'''simple docstring'''




								self.insert_nth(len(self      )		,	_UpperCAmelCase      )



				def       SCREAMING_SNAKE_CASE       (      self      :		Dict		,	_UpperCAmelCase      :		Any      )  ->    None:





								'''simple docstring'''




								self.insert_nth(0		,	_UpperCAmelCase      )



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]		,	_UpperCAmelCase      :		int		,	_UpperCAmelCase      :		Any      )  ->    None:





								'''simple docstring'''




								if index < 0 or index > len(self      ):
												raise IndexError("""list index out of range."""      )
								_lowerCAmelCase    :    Optional[int]										=      Node(_UpperCAmelCase      )
								if self.head is None:
												_lowerCAmelCase    :    Tuple										=      new_node  # first node points itself
												_lowerCAmelCase    :    int										=      new_node
								elif index == 0:  # insert at head
												_lowerCAmelCase    :    Tuple										=      self.head
												_lowerCAmelCase    :    int										=      new_node
								else:
												_lowerCAmelCase    :    Union[str, Any]										=      self.head
												for _ in range(index - 1      ):
																_lowerCAmelCase    :    Optional[Any]										=      temp.next
												_lowerCAmelCase    :    Optional[int]										=      temp.next
												_lowerCAmelCase    :    Optional[int]										=      new_node
												if index == len(self      ) - 1:  # insert at tail
																_lowerCAmelCase    :    Optional[Any]										=      new_node



				def       SCREAMING_SNAKE_CASE       (      self      :		Dict      )  ->    str:





								'''simple docstring'''




								return self.delete_nth(0      )



				def       SCREAMING_SNAKE_CASE       (      self      :		List[str]      )  ->    Any:





								'''simple docstring'''




								return self.delete_nth(len(self      ) - 1      )



				def       SCREAMING_SNAKE_CASE       (      self      :		str		,	_UpperCAmelCase      :		int = 0      )  ->    Any:





								'''simple docstring'''




								if not 0 <= index < len(self      ):
												raise IndexError("""list index out of range."""      )
								_lowerCAmelCase    :    Optional[Any]										=      self.head
								if self.head == self.tail:  # just one node
												_lowerCAmelCase    :    Optional[Any]										=      None
								elif index == 0:  # delete head node
												_lowerCAmelCase    :    str										=      self.tail.next.next
												_lowerCAmelCase    :    List[str]										=      self.head.next
								else:
												_lowerCAmelCase    :    List[str]										=      self.head
												for _ in range(index - 1      ):
																_lowerCAmelCase    :    List[Any]										=      temp.next
												_lowerCAmelCase    :    List[str]										=      temp.next
												_lowerCAmelCase    :    Dict										=      temp.next.next
												if index == len(self      ) - 1:  # delete at tail
																_lowerCAmelCase    :    Optional[int]										=      temp
								return delete_node.data



				def       SCREAMING_SNAKE_CASE       (      self      :		int      )  ->    bool:





								'''simple docstring'''




								return len(self      ) == 0






def 							_UpperCAmelCase  ():
				'''simple docstring'''
				_lowerCAmelCase    :    Dict										=      CircularLinkedList()
				assert len(UpperCamelCase_       ) == 0
				assert circular_linked_list.is_empty() is True
				assert str(UpperCamelCase_       ) == ""

				try:
								circular_linked_list.delete_front()
								raise AssertionError  # This should not happen
				except IndexError:
								assert True  # This should happen

				try:
								circular_linked_list.delete_tail()
								raise AssertionError  # This should not happen
				except IndexError:
								assert True  # This should happen

				try:
								circular_linked_list.delete_nth(-1       )
								raise AssertionError
				except IndexError:
								assert True

				try:
								circular_linked_list.delete_nth(0       )
								raise AssertionError
				except IndexError:
								assert True

				assert circular_linked_list.is_empty() is True
				for i in range(5       ):
								assert len(UpperCamelCase_       ) == i
								circular_linked_list.insert_nth(UpperCamelCase_      ,    i + 1       )
				assert str(UpperCamelCase_       ) == "->".join(str(UpperCamelCase_       ) for i in range(1      ,    6       )       )

				circular_linked_list.insert_tail(6       )
				assert str(UpperCamelCase_       ) == "->".join(str(UpperCamelCase_       ) for i in range(1      ,    7       )       )
				circular_linked_list.insert_head(0       )
				assert str(UpperCamelCase_       ) == "->".join(str(UpperCamelCase_       ) for i in range(0      ,    7       )       )

				assert circular_linked_list.delete_front() == 0
				assert circular_linked_list.delete_tail() == 6
				assert str(UpperCamelCase_       ) == "->".join(str(UpperCamelCase_       ) for i in range(1      ,    6       )       )
				assert circular_linked_list.delete_nth(2       ) == 3

				circular_linked_list.insert_nth(2      ,    3       )
				assert str(UpperCamelCase_       ) == "->".join(str(UpperCamelCase_       ) for i in range(1      ,    6       )       )

				assert circular_linked_list.is_empty() is False


if __name__ == "__main__":
	import doctest

	doctest.testmod()

