



import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin






@custom_tokenizers
class     __a			(  SCREAMING_SNAKE_CASE						,     unittest.TestCase					):
	SCREAMING_SNAKE_CASE  	=							BertJapaneseTokenizer
	SCREAMING_SNAKE_CASE  	=							False
	SCREAMING_SNAKE_CASE  	=							True



	def 		UpperCamelCase				(    self				:     List[Any])->				Optional[int]:
						super().setUp()

						__lowerCAmelCase						 =[
						    """[UNK]""",
						    """[CLS]""",
						    """[SEP]""",
						    """こんにちは""",
						    """こん""",
						    """にちは""",
						    """ばんは""",
						    """##こん""",
						    """##にちは""",
						    """##ばんは""",
						    """世界""",
						    """##世界""",
						    """、""",
						    """##、""",
						    """。""",
						    """##。""",
						]

						__lowerCAmelCase						 =os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["""vocab_file"""])
						with open(self.vocab_file					,						"""w"""					,						encoding="""utf-8""") as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))



	def 		UpperCamelCase				(    self				:     Any					,						snake_case_				:     Optional[Any])->				Union[str, Any]:
						__lowerCAmelCase						 ="""こんにちは、世界。 \nこんばんは、世界。"""
						__lowerCAmelCase						 ="""こんにちは 、 世界 。 こんばんは 、 世界 。"""
						return input_text, output_text



	def 		UpperCamelCase				(    self				:     Optional[int]					,						snake_case_				:     Optional[int])->				Optional[int]:
						__lowerCAmelCase ,						__lowerCAmelCase						 =self.get_input_output_texts(snake_case_)
						__lowerCAmelCase						 =tokenizer.encode(snake_case_					,						add_special_tokens=snake_case_)
						__lowerCAmelCase						 =tokenizer.decode(snake_case_					,						clean_up_tokenization_spaces=snake_case_)
						return text, ids



	def 		UpperCamelCase				(    self				:     List[Any])->				Union[str, Any]:
						pass  # TODO add if relevant



	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Union[str, Any]:
						pass  # TODO add if relevant



	def 		UpperCamelCase				(    self				:     Dict)->				List[str]:
						pass  # TODO add if relevant



	def 		UpperCamelCase				(    self				:     int)->				List[str]:
						__lowerCAmelCase						 =self.tokenizer_class(self.vocab_file)

						__lowerCAmelCase						 =tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。""")
						self.assertListEqual(snake_case_					,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_)					,						[3, 12, 10, 14, 4, 9, 12, 10, 14])



	def 		UpperCamelCase				(    self				:     List[Any])->				List[str]:
						__lowerCAmelCase						 =self.tokenizer_class(self.vocab_file					,						word_tokenizer_type="""mecab""")
						self.assertIsNotNone(snake_case_)

						__lowerCAmelCase						 ="""こんにちは、世界。\nこんばんは、世界。"""
						__lowerCAmelCase						 =tokenizer.tokenize(snake_case_)
						self.assertListEqual(snake_case_					,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_)					,						[3, 12, 10, 14, 4, 9, 12, 10, 14])

						__lowerCAmelCase						 =os.path.join(self.tmpdirname					,						"""tokenizer.bin""")
						with open(snake_case_					,						"""wb""") as handle:
											pickle.dump(snake_case_					,						snake_case_)

						with open(snake_case_					,						"""rb""") as handle:
											__lowerCAmelCase						 =pickle.load(snake_case_)

						__lowerCAmelCase						 =tokenizer_new.tokenize(snake_case_)

						self.assertListEqual(snake_case_					,						snake_case_)



	def 		UpperCamelCase				(    self				:     Any)->				Union[str, Any]:
						__lowerCAmelCase						 =MecabTokenizer(mecab_dic="""ipadic""")

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,						)



	def 		UpperCamelCase				(    self				:     Tuple)->				str:
						try:
											__lowerCAmelCase						 =MecabTokenizer(mecab_dic="""unidic_lite""")
						except ModuleNotFoundError:
											return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,						)



	def 		UpperCamelCase				(    self				:     List[str])->				int:
						try:
											__lowerCAmelCase						 =MecabTokenizer(mecab_dic="""unidic""")
						except ModuleNotFoundError:
											return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,						)



	def 		UpperCamelCase				(    self				:     List[str])->				Dict:
						__lowerCAmelCase						 =MecabTokenizer(do_lower_case=snake_case_					,						mecab_dic="""ipadic""")

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,						)



	def 		UpperCamelCase				(    self				:     str)->				int:
						try:
											__lowerCAmelCase						 =MecabTokenizer(
											    do_lower_case=snake_case_					,						normalize_text=snake_case_					,						mecab_option="""-d /usr/local/lib/mecab/dic/jumandic""")
						except RuntimeError:
											# if dict doesn't exist in the system, previous code raises this error.
											return

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,						)



	def 		UpperCamelCase				(    self				:     Optional[Any])->				List[str]:
						__lowerCAmelCase						 =MecabTokenizer(normalize_text=snake_case_					,						mecab_dic="""ipadic""")

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]					,						)



	@require_sudachi
	def 		UpperCamelCase				(    self				:     Any)->				List[Any]:
						__lowerCAmelCase						 =self.tokenizer_class(self.vocab_file					,						word_tokenizer_type="""sudachi""")
						self.assertIsNotNone(snake_case_)

						__lowerCAmelCase						 ="""こんにちは、世界。\nこんばんは、世界。"""
						__lowerCAmelCase						 =tokenizer.tokenize(snake_case_)
						self.assertListEqual(snake_case_					,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_)					,						[3, 12, 10, 14, 4, 9, 12, 10, 14])

						__lowerCAmelCase						 =os.path.join(self.tmpdirname					,						"""tokenizer.bin""")
						with open(snake_case_					,						"""wb""") as handle:
											pickle.dump(snake_case_					,						snake_case_)

						with open(snake_case_					,						"""rb""") as handle:
											__lowerCAmelCase						 =pickle.load(snake_case_)

						__lowerCAmelCase						 =tokenizer_new.tokenize(snake_case_)

						self.assertListEqual(snake_case_					,						snake_case_)



	@require_sudachi
	def 		UpperCamelCase				(    self				:     Dict)->				List[str]:
						__lowerCAmelCase						 =SudachiTokenizer(sudachi_dict_type="""core""")

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]					,						)



	@require_sudachi
	def 		UpperCamelCase				(    self				:     Optional[Any])->				List[Any]:
						__lowerCAmelCase						 =SudachiTokenizer(sudachi_dict_type="""core"""					,						sudachi_split_mode="""A""")

						self.assertListEqual(tokenizer.tokenize("""外国人参政権""")					,						["""外国""", """人""", """参政""", """権"""])



	@require_sudachi
	def 		UpperCamelCase				(    self				:     str)->				List[str]:
						__lowerCAmelCase						 =SudachiTokenizer(sudachi_dict_type="""core"""					,						sudachi_split_mode="""B""")

						self.assertListEqual(tokenizer.tokenize("""外国人参政権""")					,						["""外国人""", """参政権"""])



	@require_sudachi
	def 		UpperCamelCase				(    self				:     int)->				str:
						__lowerCAmelCase						 =SudachiTokenizer(sudachi_dict_type="""core"""					,						sudachi_split_mode="""C""")

						self.assertListEqual(tokenizer.tokenize("""外国人参政権""")					,						["""外国人参政権"""])



	@require_sudachi
	def 		UpperCamelCase				(    self				:     Tuple)->				Union[str, Any]:
						__lowerCAmelCase						 =SudachiTokenizer(do_lower_case=snake_case_					,						sudachi_dict_type="""core""")

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]					,						)



	@require_sudachi
	def 		UpperCamelCase				(    self				:     Optional[int])->				Dict:
						__lowerCAmelCase						 =SudachiTokenizer(normalize_text=snake_case_					,						sudachi_dict_type="""core""")

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]					,						)



	@require_sudachi
	def 		UpperCamelCase				(    self				:     Union[str, Any])->				Tuple:
						__lowerCAmelCase						 =SudachiTokenizer(trim_whitespace=snake_case_					,						sudachi_dict_type="""core""")

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]					,						)



	@require_jumanpp
	def 		UpperCamelCase				(    self				:     Optional[Any])->				str:
						__lowerCAmelCase						 =self.tokenizer_class(self.vocab_file					,						word_tokenizer_type="""jumanpp""")
						self.assertIsNotNone(snake_case_)

						__lowerCAmelCase						 ="""こんにちは、世界。\nこんばんは、世界。"""
						__lowerCAmelCase						 =tokenizer.tokenize(snake_case_)
						self.assertListEqual(snake_case_					,						["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""])
						self.assertListEqual(tokenizer.convert_tokens_to_ids(snake_case_)					,						[3, 12, 10, 14, 4, 9, 12, 10, 14])

						__lowerCAmelCase						 =os.path.join(self.tmpdirname					,						"""tokenizer.bin""")
						with open(snake_case_					,						"""wb""") as handle:
											pickle.dump(snake_case_					,						snake_case_)

						with open(snake_case_					,						"""rb""") as handle:
											__lowerCAmelCase						 =pickle.load(snake_case_)

						__lowerCAmelCase						 =tokenizer_new.tokenize(snake_case_)

						self.assertListEqual(snake_case_					,						snake_case_)



	@require_jumanpp
	def 		UpperCamelCase				(    self				:     List[str])->				Dict:
						__lowerCAmelCase						 =JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,						)



	@require_jumanpp
	def 		UpperCamelCase				(    self				:     Tuple)->				Union[str, Any]:
						__lowerCAmelCase						 =JumanppTokenizer(do_lower_case=snake_case_)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,						)



	@require_jumanpp
	def 		UpperCamelCase				(    self				:     Tuple)->				Dict:
						__lowerCAmelCase						 =JumanppTokenizer(normalize_text=snake_case_)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]					,						)



	@require_jumanpp
	def 		UpperCamelCase				(    self				:     Dict)->				Optional[Any]:
						__lowerCAmelCase						 =JumanppTokenizer(trim_whitespace=snake_case_)

						self.assertListEqual(
						    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """)					,						["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]					,						)



	@require_jumanpp
	def 		UpperCamelCase				(    self				:     List[Any])->				Union[str, Any]:
						__lowerCAmelCase						 =JumanppTokenizer()

						self.assertListEqual(
						    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。""")					,						["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]					,						)



	def 		UpperCamelCase				(    self				:     Optional[int])->				Union[str, Any]:
						__lowerCAmelCase						 =["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

						__lowerCAmelCase						 ={}
						for i, token in enumerate(snake_case_):
											__lowerCAmelCase						 =i
						__lowerCAmelCase						 =WordpieceTokenizer(vocab=snake_case_					,						unk_token="""[UNK]""")

						self.assertListEqual(tokenizer.tokenize("""""")					,						[])

						self.assertListEqual(tokenizer.tokenize("""こんにちは""")					,						["""こんにちは"""])

						self.assertListEqual(tokenizer.tokenize("""こんばんは""")					,						["""こん""", """##ばんは"""])

						self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは""")					,						["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""])



	def 		UpperCamelCase				(    self				:     Optional[int])->				List[str]:
						__lowerCAmelCase						 =BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp""")
						__lowerCAmelCase						 =tokenizer.subword_tokenizer

						__lowerCAmelCase						 =subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。""")
						self.assertListEqual(snake_case_					,						["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""])

						__lowerCAmelCase						 =subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは""")
						self.assertListEqual(snake_case_					,						["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""])



	def 		UpperCamelCase				(    self				:     Dict)->				Union[str, Any]:
						__lowerCAmelCase						 =self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese""")

						__lowerCAmelCase						 =tokenizer.encode("""ありがとう。"""					,						add_special_tokens=snake_case_)
						__lowerCAmelCase						 =tokenizer.encode("""どういたしまして。"""					,						add_special_tokens=snake_case_)

						__lowerCAmelCase						 =tokenizer.build_inputs_with_special_tokens(snake_case_)
						__lowerCAmelCase						 =tokenizer.build_inputs_with_special_tokens(snake_case_					,						snake_case_)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class     __a			(  SCREAMING_SNAKE_CASE						,     unittest.TestCase					):
	SCREAMING_SNAKE_CASE  	=							BertJapaneseTokenizer
	SCREAMING_SNAKE_CASE  	=							False



	def 		UpperCamelCase				(    self				:     str)->				List[Any]:
						super().setUp()

						__lowerCAmelCase						 =["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						__lowerCAmelCase						 =os.path.join(self.tmpdirname					,						VOCAB_FILES_NAMES["""vocab_file"""])
						with open(self.vocab_file					,						"""w"""					,						encoding="""utf-8""") as vocab_writer:
											vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]))



	def 		UpperCamelCase				(    self				:     Optional[int]					,						**snake_case_				:     Any)->				int:
						return BertJapaneseTokenizer.from_pretrained(self.tmpdirname					,						subword_tokenizer_type="""character"""					,						**snake_case_)



	def 		UpperCamelCase				(    self				:     Optional[int]					,						snake_case_				:     Optional[int])->				Dict:
						__lowerCAmelCase						 ="""こんにちは、世界。 \nこんばんは、世界。"""
						__lowerCAmelCase						 ="""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
						return input_text, output_text



	def 		UpperCamelCase				(    self				:     str)->				Union[str, Any]:
						pass  # TODO add if relevant



	def 		UpperCamelCase				(    self				:     List[str])->				int:
						pass  # TODO add if relevant



	def 		UpperCamelCase				(    self				:     Any)->				List[str]:
						pass  # TODO add if relevant



	def 		UpperCamelCase				(    self				:     Tuple)->				Union[str, Any]:
						__lowerCAmelCase						 =self.tokenizer_class(self.vocab_file					,						subword_tokenizer_type="""character""")

						__lowerCAmelCase						 =tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。""")
						self.assertListEqual(
						    snake_case_					,						["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""])
						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(snake_case_)					,						[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12])



	def 		UpperCamelCase				(    self				:     Any)->				Optional[int]:
						__lowerCAmelCase						 =["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

						__lowerCAmelCase						 ={}
						for i, token in enumerate(snake_case_):
											__lowerCAmelCase						 =i
						__lowerCAmelCase						 =CharacterTokenizer(vocab=snake_case_					,						unk_token="""[UNK]""")

						self.assertListEqual(tokenizer.tokenize("""""")					,						[])

						self.assertListEqual(tokenizer.tokenize("""こんにちは""")					,						["""こ""", """ん""", """に""", """ち""", """は"""])

						self.assertListEqual(tokenizer.tokenize("""こんにちほ""")					,						["""こ""", """ん""", """に""", """ち""", """[UNK]"""])



	def 		UpperCamelCase				(    self				:     List[str])->				Dict:
						__lowerCAmelCase						 =self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char""")

						__lowerCAmelCase						 =tokenizer.encode("""ありがとう。"""					,						add_special_tokens=snake_case_)
						__lowerCAmelCase						 =tokenizer.encode("""どういたしまして。"""					,						add_special_tokens=snake_case_)

						__lowerCAmelCase						 =tokenizer.build_inputs_with_special_tokens(snake_case_)
						__lowerCAmelCase						 =tokenizer.build_inputs_with_special_tokens(snake_case_					,						snake_case_)

						# 2 is for "[CLS]", 3 is for "[SEP]"
						assert encoded_sentence == [2] + text + [3]
						assert encoded_pair == [2] + text + [3] + text_a + [3]






@custom_tokenizers
class     __a			(  unittest.TestCase					):



	def 		UpperCamelCase				(    self				:     Any)->				Optional[int]:
						__lowerCAmelCase						 ="""cl-tohoku/bert-base-japanese"""
						__lowerCAmelCase						 =AutoTokenizer.from_pretrained(snake_case_)
						self.assertIsInstance(snake_case_					,						snake_case_)






class     __a			(  unittest.TestCase					):



	def 		UpperCamelCase				(    self				:     Dict)->				Tuple:
						__lowerCAmelCase						 ="""cl-tohoku/bert-base-japanese"""
						with self.assertLogs("""transformers"""					,						level="""WARNING""") as cm:
											BertTokenizer.from_pretrained(snake_case_)
											self.assertTrue(
											    cm.records[0].message.startswith(
											        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
											        """ is called from."""))
						__lowerCAmelCase						 ="""bert-base-cased"""
						with self.assertLogs("""transformers"""					,						level="""WARNING""") as cm:
											BertJapaneseTokenizer.from_pretrained(snake_case_)
											self.assertTrue(
											    cm.records[0].message.startswith(
											        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
											        """ is called from."""))









from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowercase_													=      {
    '''configuration_albert''': ['''ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''AlbertConfig''', '''AlbertOnnxConfig'''],
}

try:
				if not is_sentencepiece_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      ['''AlbertTokenizer''']

try:
				if not is_tokenizers_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      ['''AlbertTokenizerFast''']

try:
				if not is_torch_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''AlbertForMaskedLM''',
				    '''AlbertForMultipleChoice''',
				    '''AlbertForPreTraining''',
				    '''AlbertForQuestionAnswering''',
				    '''AlbertForSequenceClassification''',
				    '''AlbertForTokenClassification''',
				    '''AlbertModel''',
				    '''AlbertPreTrainedModel''',
				    '''load_tf_weights_in_albert''',
				]

try:
				if not is_tf_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST''',
				    '''TFAlbertForMaskedLM''',
				    '''TFAlbertForMultipleChoice''',
				    '''TFAlbertForPreTraining''',
				    '''TFAlbertForQuestionAnswering''',
				    '''TFAlbertForSequenceClassification''',
				    '''TFAlbertForTokenClassification''',
				    '''TFAlbertMainLayer''',
				    '''TFAlbertModel''',
				    '''TFAlbertPreTrainedModel''',
				]

try:
				if not is_flax_available():
								raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
				pass
else:
				lowercase_													=      [
				    '''FlaxAlbertForMaskedLM''',
				    '''FlaxAlbertForMultipleChoice''',
				    '''FlaxAlbertForPreTraining''',
				    '''FlaxAlbertForQuestionAnswering''',
				    '''FlaxAlbertForSequenceClassification''',
				    '''FlaxAlbertForTokenClassification''',
				    '''FlaxAlbertModel''',
				    '''FlaxAlbertPreTrainedModel''',
				]

if TYPE_CHECKING:
				from .configuration_albert import ALBERT_PRETRAINED_CONFIG_ARCHIVE_MAP, AlbertConfig, AlbertOnnxConfig

				try:
								if not is_sentencepiece_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert import AlbertTokenizer

				try:
								if not is_tokenizers_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .tokenization_albert_fast import AlbertTokenizerFast

				try:
								if not is_torch_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_albert import (
								    ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    AlbertForMaskedLM,
								    AlbertForMultipleChoice,
								    AlbertForPreTraining,
								    AlbertForQuestionAnswering,
								    AlbertForSequenceClassification,
								    AlbertForTokenClassification,
								    AlbertModel,
								    AlbertPreTrainedModel,
								    load_tf_weights_in_albert,
								)

				try:
								if not is_tf_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_tf_albert import (
								    TF_ALBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
								    TFAlbertForMaskedLM,
								    TFAlbertForMultipleChoice,
								    TFAlbertForPreTraining,
								    TFAlbertForQuestionAnswering,
								    TFAlbertForSequenceClassification,
								    TFAlbertForTokenClassification,
								    TFAlbertMainLayer,
								    TFAlbertModel,
								    TFAlbertPreTrainedModel,
								)

				try:
								if not is_flax_available():
												raise OptionalDependencyNotAvailable()
				except OptionalDependencyNotAvailable:
								pass
				else:
								from .modeling_flax_albert import (
								    FlaxAlbertForMaskedLM,
								    FlaxAlbertForMultipleChoice,
								    FlaxAlbertForPreTraining,
								    FlaxAlbertForQuestionAnswering,
								    FlaxAlbertForSequenceClassification,
								    FlaxAlbertForTokenClassification,
								    FlaxAlbertModel,
								    FlaxAlbertPreTrainedModel,
								)
else:
				import sys

				lowercase_													=      _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)




