


'''simple docstring'''



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin


@custom_tokenizers
class 		_snake_case					(     a__       ,   unittest.TestCase	):
					lowerCAmelCase      :int   =					BertJapaneseTokenizer
					lowerCAmelCase      :Dict   =					False
					lowerCAmelCase      :int   =					True




					def 				snake_case__						(      self):
								super().setUp()

								UpperCAmelCase__  :					Any        =     [
								    """[UNK]""",
								    """[CLS]""",
								    """[SEP]""",
								    """こんにちは""",
								    """こん""",
								    """にちは""",
								    """ばんは""",
								    """##こん""",
								    """##にちは""",
								    """##ばんは""",
								    """世界""",
								    """##世界""",
								    """、""",
								    """##、""",
								    """。""",
								    """##。""",
								]

								UpperCAmelCase__  :					str        =     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES["""vocab_file"""])
								with open(self.vocab_file							,							"""w"""							,							encoding="""utf-8""") as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					Union[str, Any]        =     """こんにちは、世界。 \nこんばんは、世界。"""
								UpperCAmelCase__  :					Union[str, Any]        =     """こんにちは 、 世界 。 こんばんは 、 世界 。"""
								return input_text, output_text




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__   ,       UpperCAmelCase__  :					List[Any]        =     self.get_input_output_texts(_lowerCamelCase)
								UpperCAmelCase__  :					Dict        =     tokenizer.encode(_lowerCamelCase							,							add_special_tokens=_lowerCamelCase)
								UpperCAmelCase__  :					int        =     tokenizer.decode(_lowerCamelCase							,							clean_up_tokenization_spaces=_lowerCamelCase)
								return text, ids




					def 				snake_case__						(      self):
								pass  # TODO add if relevant




					def 				snake_case__						(      self):
								pass  # TODO add if relevant




					def 				snake_case__						(      self):
								pass  # TODO add if relevant




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     self.tokenizer_class(self.vocab_file)

								UpperCAmelCase__  :					str        =     tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。""")
								self.assertListEqual(_lowerCamelCase							,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     self.tokenizer_class(self.vocab_file							,							word_tokenizer_type="""mecab""")
								self.assertIsNotNone(_lowerCamelCase)

								UpperCAmelCase__  :					int        =     """こんにちは、世界。\nこんばんは、世界。"""
								UpperCAmelCase__  :					str        =     tokenizer.tokenize(_lowerCamelCase)
								self.assertListEqual(_lowerCamelCase							,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

								UpperCAmelCase__  :					Dict        =     os.path.join(self.tmpdirname							,							"""tokenizer.bin""")
								with open(_lowerCamelCase							,							"""wb""") as handle:
											pickle.dump(_lowerCamelCase							,							_lowerCamelCase)

								with open(_lowerCamelCase							,							"""rb""") as handle:
											UpperCAmelCase__  :					Dict        =     pickle.load(_lowerCamelCase)

								UpperCAmelCase__  :					Optional[int]        =     tokenizer_new.tokenize(_lowerCamelCase)

								self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Any        =     MecabTokenizer(mecab_dic="""ipadic""")

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,							)




					def 				snake_case__						(      self):

								try:
											UpperCAmelCase__  :					Optional[int]        =     MecabTokenizer(mecab_dic="""unidic_lite""")
								except ModuleNotFoundError:
											return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,							)




					def 				snake_case__						(      self):

								try:
											UpperCAmelCase__  :					Union[str, Any]        =     MecabTokenizer(mecab_dic="""unidic""")
								except ModuleNotFoundError:
											return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,							)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     MecabTokenizer(do_lower_case=_lowerCamelCase							,							mecab_dic="""ipadic""")

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,							)




					def 				snake_case__						(      self):

								try:
											UpperCAmelCase__  :					Tuple        =     MecabTokenizer(
											    do_lower_case=_lowerCamelCase							,							normalize_text=_lowerCamelCase							,							mecab_option="""-d /usr/local/lib/mecab/dic/jumandic""")
								except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,							)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     MecabTokenizer(normalize_text=_lowerCamelCase							,							mecab_dic="""ipadic""")

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]							,							)




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Any        =     self.tokenizer_class(self.vocab_file							,							word_tokenizer_type="""sudachi""")
								self.assertIsNotNone(_lowerCamelCase)

								UpperCAmelCase__  :					int        =     """こんにちは、世界。\nこんばんは、世界。"""
								UpperCAmelCase__  :					int        =     tokenizer.tokenize(_lowerCamelCase)
								self.assertListEqual(_lowerCamelCase							,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

								UpperCAmelCase__  :					List[str]        =     os.path.join(self.tmpdirname							,							"""tokenizer.bin""")
								with open(_lowerCamelCase							,							"""wb""") as handle:
											pickle.dump(_lowerCamelCase							,							_lowerCamelCase)

								with open(_lowerCamelCase							,							"""rb""") as handle:
											UpperCAmelCase__  :					Dict        =     pickle.load(_lowerCamelCase)

								UpperCAmelCase__  :					Optional[int]        =     tokenizer_new.tokenize(_lowerCamelCase)

								self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Tuple        =     SudachiTokenizer(sudachi_dict_type="""core""")

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]							,							)




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     SudachiTokenizer(sudachi_dict_type="""core"""							,							sudachi_split_mode="""A""")

								self.assertListEqual(tokenizer.tokenize("""外国人参政権""")							,							["""外国""", """人""", """参政""", """権"""])




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     SudachiTokenizer(sudachi_dict_type="""core"""							,							sudachi_split_mode="""B""")

								self.assertListEqual(tokenizer.tokenize("""外国人参政権""")							,							["""外国人""", """参政権"""])




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Union[str, Any]        =     SudachiTokenizer(sudachi_dict_type="""core"""							,							sudachi_split_mode="""C""")

								self.assertListEqual(tokenizer.tokenize("""外国人参政権""")							,							["""外国人参政権"""])




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					int        =     SudachiTokenizer(do_lower_case=_lowerCamelCase							,							sudachi_dict_type="""core""")

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]							,							)




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[Any]        =     SudachiTokenizer(normalize_text=_lowerCamelCase							,							sudachi_dict_type="""core""")

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]							,							)




					@require_sudachi
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					str        =     SudachiTokenizer(trim_whitespace=_lowerCamelCase							,							sudachi_dict_type="""core""")

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]							,							)




					@require_jumanpp
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     self.tokenizer_class(self.vocab_file							,							word_tokenizer_type="""jumanpp""")
								self.assertIsNotNone(_lowerCamelCase)

								UpperCAmelCase__  :					Tuple        =     """こんにちは、世界。\nこんばんは、世界。"""
								UpperCAmelCase__  :					Optional[Any]        =     tokenizer.tokenize(_lowerCamelCase)
								self.assertListEqual(_lowerCamelCase							,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
								self.assertListEqual(tokenizer.convert_tokens_to_ids(_lowerCamelCase)							,							[3, 12, 10, 14, 4, 9, 12, 10, 14])

								UpperCAmelCase__  :					List[str]        =     os.path.join(self.tmpdirname							,							"""tokenizer.bin""")
								with open(_lowerCamelCase							,							"""wb""") as handle:
											pickle.dump(_lowerCamelCase							,							_lowerCamelCase)

								with open(_lowerCamelCase							,							"""rb""") as handle:
											UpperCAmelCase__  :					int        =     pickle.load(_lowerCamelCase)

								UpperCAmelCase__  :					int        =     tokenizer_new.tokenize(_lowerCamelCase)

								self.assertListEqual(_lowerCamelCase							,							_lowerCamelCase)




					@require_jumanpp
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,							)




					@require_jumanpp
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					str        =     JumanppTokenizer(do_lower_case=_lowerCamelCase)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,							)




					@require_jumanpp
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					str        =     JumanppTokenizer(normalize_text=_lowerCamelCase)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]							,							)




					@require_jumanpp
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Dict        =     JumanppTokenizer(trim_whitespace=_lowerCamelCase)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)							,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]							,							)




					@require_jumanpp
					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Union[str, Any]        =     JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。""")							,							["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]							,							)




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					int        =     ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

								UpperCAmelCase__  :					Optional[Any]        =     {}
								for i, token in enumerate(_lowerCamelCase):
											UpperCAmelCase__  :					str        =     i
								UpperCAmelCase__  :					Optional[Any]        =     WordpieceTokenizer(vocab=_lowerCamelCase							,							unk_token="""[UNK]""")

								self.assertListEqual(tokenizer.tokenize("""""")							,							[])

								self.assertListEqual(tokenizer.tokenize("""こんにちは""")							,							["""こんにちは"""])

								self.assertListEqual(tokenizer.tokenize("""こんばんは""")							,							["""こん""", """##ばんは"""])

								self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは""")							,							["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[Any]        =     BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp""")
								UpperCAmelCase__  :					List[str]        =     tokenizer.subword_tokenizer

								UpperCAmelCase__  :					Optional[int]        =     subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。""")
								self.assertListEqual(_lowerCamelCase							,							["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""])

								UpperCAmelCase__  :					str        =     subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは""")
								self.assertListEqual(_lowerCamelCase							,							["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[int]        =     self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese""")

								UpperCAmelCase__  :					Tuple        =     tokenizer.encode("""ありがとう。"""							,							add_special_tokens=_lowerCamelCase)
								UpperCAmelCase__  :					Dict        =     tokenizer.encode("""どういたしまして。"""							,							add_special_tokens=_lowerCamelCase)

								UpperCAmelCase__  :					List[str]        =     tokenizer.build_inputs_with_special_tokens(_lowerCamelCase)
								UpperCAmelCase__  :					List[str]        =     tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							,							_lowerCamelCase)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class 		_snake_case					(     a__       ,   unittest.TestCase	):
					lowerCAmelCase      :List[str]   =					BertJapaneseTokenizer
					lowerCAmelCase      :List[str]   =					False




					def 				snake_case__						(      self):
								super().setUp()

								UpperCAmelCase__  :					Any        =     ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

								UpperCAmelCase__  :					str        =     os.path.join(self.tmpdirname							,							VOCAB_FILES_NAMES["""vocab_file"""])
								with open(self.vocab_file							,							"""w"""							,							encoding="""utf-8""") as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))




					def 				snake_case__						(      self							,							**_lowerCamelCase):
								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname							,							subword_tokenizer_type="""character"""							,							**_lowerCamelCase)




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					List[Any]        =     """こんにちは、世界。 \nこんばんは、世界。"""
								UpperCAmelCase__  :					Union[str, Any]        =     """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
								return input_text, output_text




					def 				snake_case__						(      self):
								pass  # TODO add if relevant




					def 				snake_case__						(      self):
								pass  # TODO add if relevant




					def 				snake_case__						(      self):
								pass  # TODO add if relevant




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[str]        =     self.tokenizer_class(self.vocab_file							,							subword_tokenizer_type="""character""")

								UpperCAmelCase__  :					Optional[Any]        =     tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。""")
								self.assertListEqual(
								    _lowerCamelCase							,							["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""])
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(_lowerCamelCase)							,							[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					int        =     ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

								UpperCAmelCase__  :					Dict        =     {}
								for i, token in enumerate(_lowerCamelCase):
											UpperCAmelCase__  :					Tuple        =     i
								UpperCAmelCase__  :					List[str]        =     CharacterTokenizer(vocab=_lowerCamelCase							,							unk_token="""[UNK]""")

								self.assertListEqual(tokenizer.tokenize("""""")							,							[])

								self.assertListEqual(tokenizer.tokenize("""こんにちは""")							,							["""こ""", """ん""", """に""", """ち""", """は"""])

								self.assertListEqual(tokenizer.tokenize("""こんにちほ""")							,							["""こ""", """ん""", """に""", """ち""", """[UNK]"""])




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					List[Any]        =     self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char""")

								UpperCAmelCase__  :					Optional[int]        =     tokenizer.encode("""ありがとう。"""							,							add_special_tokens=_lowerCamelCase)
								UpperCAmelCase__  :					Any        =     tokenizer.encode("""どういたしまして。"""							,							add_special_tokens=_lowerCamelCase)

								UpperCAmelCase__  :					str        =     tokenizer.build_inputs_with_special_tokens(_lowerCamelCase)
								UpperCAmelCase__  :					Optional[int]        =     tokenizer.build_inputs_with_special_tokens(_lowerCamelCase							,							_lowerCamelCase)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]


@custom_tokenizers
class 		_snake_case					(     unittest.TestCase	):




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[Any]        =     """cl-tohoku/bert-base-japanese"""
								UpperCAmelCase__  :					List[Any]        =     AutoTokenizer.from_pretrained(_lowerCamelCase)
								self.assertIsInstance(_lowerCamelCase							,							_lowerCamelCase)


class 		_snake_case					(     unittest.TestCase	):




					def 				snake_case__						(      self):
								UpperCAmelCase__  :					Optional[Any]        =     """cl-tohoku/bert-base-japanese"""
								with self.assertLogs("""transformers"""							,							level="""WARNING""") as cm:
											BertTokenizer.from_pretrained(_lowerCamelCase)
											self.assertTrue(
											    cm.records[0].message.startswith(
											        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
											        """ is called from."""))
								UpperCAmelCase__  :					Tuple        =     """bert-base-cased"""
								with self.assertLogs("""transformers"""							,							level="""WARNING""") as cm:
											BertJapaneseTokenizer.from_pretrained(_lowerCamelCase)
											self.assertTrue(
											    cm.records[0].message.startswith(
											        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
											        """ is called from."""))



'''simple docstring'''



from collections import Counter
from pathlib import Path
from typing import Optional, Tuple

import yaml


class 		_snake_case					(     yaml.SafeLoader	):




					def 				snake_case__						(      self							,							_lowerCamelCase):
								UpperCAmelCase__  :					Dict        =     [self.constructed_objects[key_node] for key_node, _ in node.value]
								UpperCAmelCase__  :					List[str]        =     [tuple(_lowerCamelCase) if isinstance(_lowerCamelCase							,							_lowerCamelCase) else key for key in keys]
								UpperCAmelCase__  :					List[str]        =     Counter(_lowerCamelCase)
								UpperCAmelCase__  :					Any        =     [key for key in counter if counter[key] > 1]
								if duplicate_keys:
											raise TypeError(f'''Got duplicate yaml keys: {duplicate_keys}''')




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase=False):
								UpperCAmelCase__  :					List[str]        =     super().construct_mapping(_lowerCamelCase							,							deep=_lowerCamelCase)
								self._check_no_duplicates_on_constructed_node(_lowerCamelCase)
								return mapping







def     _UpperCamelCase      (		UpperCamelCase__					):
			UpperCAmelCase__  :					Tuple        =     list(readme_content.splitlines()					)
			if full_content and full_content[0] == "---" and "---" in full_content[1:]:
						UpperCAmelCase__  :					Dict        =     full_content[1:].index("""---"""					) + 1
						UpperCAmelCase__  :					List[Any]        =     """\n""".join(full_content[1:sep_idx]					)
						return yamlblock, "\n".join(full_content[sep_idx + 1 :]					)

			return None, "\n".join(UpperCamelCase__					)


class 		_snake_case					(     a__	):
					# class attributes
					lowerCAmelCase      :str   =					{'''train_eval_index'''}  # train-eval-index in the YAML metadata




					@classmethod
					def 				snake_case__						(      cls							,							_lowerCamelCase):
								with open(_lowerCamelCase							,							encoding="""utf-8""") as readme_file:
											UpperCAmelCase__   ,       UpperCAmelCase__  :					Any        =     _split_yaml_from_readme(readme_file.read())
								if yaml_string is not None:
											return cls.from_yaml_string(_lowerCamelCase)
								else:
											return cls()




					def 				snake_case__						(      self							,							_lowerCamelCase):
								if path.exists():
											with open(_lowerCamelCase							,							encoding="""utf-8""") as readme_file:
														UpperCAmelCase__  :					int        =     readme_file.read()
								else:
											UpperCAmelCase__  :					List[str]        =     None
								UpperCAmelCase__  :					Union[str, Any]        =     self._to_readme(_lowerCamelCase)
								with open(_lowerCamelCase							,							"""w"""							,							encoding="""utf-8""") as readme_file:
											readme_file.write(_lowerCamelCase)




					def 				snake_case__						(      self							,							_lowerCamelCase = None):
								if readme_content is not None:
											UpperCAmelCase__   ,       UpperCAmelCase__  :					Optional[Any]        =     _split_yaml_from_readme(_lowerCamelCase)
											UpperCAmelCase__  :					str        =     """---\n""" + self.to_yaml_string() + """---\n""" + content
								else:
											UpperCAmelCase__  :					Optional[int]        =     """---\n""" + self.to_yaml_string() + """---\n"""
								return full_content




					@classmethod
					def 				snake_case__						(      cls							,							_lowerCamelCase):
								UpperCAmelCase__  :					List[str]        =     yaml.load(_lowerCamelCase							,							Loader=_NoDuplicateSafeLoader) or {}

								# Convert the YAML keys to DatasetMetadata fields
								UpperCAmelCase__  :					List[str]        =     {
								    (key.replace("""-"""							,							"""_""") if key.replace("""-"""							,							"""_""") in cls._FIELDS_WITH_DASHES else key): value
								    for key, value in metadata_dict.items()
								}
								return cls(**_lowerCamelCase)




					def 				snake_case__						(      self):
								return yaml.safe_dump(
								    {
								        (key.replace("""_"""							,							"""-""") if key in self._FIELDS_WITH_DASHES else key): value
								        for key, value in self.items()
								    }							,							sort_keys=_lowerCamelCase							,							allow_unicode=_lowerCamelCase							,							encoding="""utf-8"""							,							).decode("""utf-8""")

__A	={
    'image-classification': [],
    'translation': [],
    'image-segmentation': [],
    'fill-mask': [],
    'automatic-speech-recognition': [],
    'token-classification': [],
    'sentence-similarity': [],
    'audio-classification': [],
    'question-answering': [],
    'summarization': [],
    'zero-shot-classification': [],
    'table-to-text': [],
    'feature-extraction': [],
    'other': [],
    'multiple-choice': [],
    'text-classification': [],
    'text-to-image': [],
    'text2text-generation': [],
    'zero-shot-image-classification': [],
    'tabular-classification': [],
    'tabular-regression': [],
    'image-to-image': [],
    'tabular-to-text': [],
    'unconditional-image-generation': [],
    'text-retrieval': [],
    'text-to-speech': [],
    'object-detection': [],
    'audio-to-audio': [],
    'text-generation': [],
    'conversational': [],
    'table-question-answering': [],
    'visual-question-answering': [],
    'image-to-text': [],
    'reinforcement-learning': [],
    'voice-activity-detection': [],
    'time-series-forecasting': [],
    'document-question-answering': [],
}


if __name__ == "__main__":
		from argparse import ArgumentParser

		__A	=ArgumentParser(usage='Validate the yaml metadata block of a README.md file.')
		ap.add_argument('readme_filepath')
		__A	=ap.parse_args()

		__A	=Path(args.readme_filepath)
		__A	=DatasetMetadata.from_readme(readme_filepath)
		print(dataset_metadata)
		dataset_metadata.to_readme(readme_filepath)