
import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin







@custom_tokenizers
class        lowerCAmelCase__      (   __lowercase		,	unittest.TestCase	):
						UpperCamelCase_							: Tuple	=		BertJapaneseTokenizer
						UpperCamelCase_							: str	=		False
						UpperCamelCase_							: Tuple	=		True

						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								super().setUp()

								_UpperCamelCase						    =	[
								    """[UNK]""",
								    """[CLS]""",
								    """[SEP]""",
								    """こんにちは""",
								    """こん""",
								    """にちは""",
								    """ばんは""",
								    """##こん""",
								    """##にちは""",
								    """##ばんは""",
								    """世界""",
								    """##世界""",
								    """、""",
								    """##、""",
								    """。""",
								    """##。""",
								]

								_UpperCamelCase						    =	os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["""vocab_file"""]				)
								with open(self.vocab_file       ,							"""w"""       ,							encoding="""utf-8"""				) as vocab_writer:
										vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]				)				)

						def  A_				(			self       ,							a				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	"""こんにちは、世界。 \nこんばんは、世界。"""
								_UpperCamelCase						    =	"""こんにちは 、 世界 。 こんばんは 、 世界 。"""
								return input_text, output_text

						def  A_				(			self       ,							a				)					->      int:


								'''simple docstring'''

								_UpperCamelCase    ,	_UpperCamelCase						    =	self.get_input_output_texts(a				)
								_UpperCamelCase						    =	tokenizer.encode(a       ,							add_special_tokens=a				)
								_UpperCamelCase						    =	tokenizer.decode(a       ,							clean_up_tokenization_spaces=a				)
								return text, ids

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								pass  # TODO add if relevant

						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								pass  # TODO add if relevant

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								pass  # TODO add if relevant

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.tokenizer_class(self.vocab_file				)

								_UpperCamelCase						    =	tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""				)
								self.assertListEqual(a       ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a				)       ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.tokenizer_class(self.vocab_file       ,							word_tokenizer_type="""mecab"""				)
								self.assertIsNotNone(a				)

								_UpperCamelCase						    =	"""こんにちは、世界。\nこんばんは、世界。"""
								_UpperCamelCase						    =	tokenizer.tokenize(a				)
								self.assertListEqual(a       ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a				)       ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]				)

								_UpperCamelCase						    =	os.path.join(self.tmpdirname       ,							"""tokenizer.bin"""				)
								with open(a       ,							"""wb"""				) as handle:
										pickle.dump(a       ,							a				)

								with open(a       ,							"""rb"""				) as handle:
										_UpperCamelCase						    =	pickle.load(a				)

								_UpperCamelCase						    =	tokenizer_new.tokenize(a				)

								self.assertListEqual(a       ,							a				)

						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	MecabTokenizer(mecab_dic="""ipadic"""				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,							)

						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								try:
										_UpperCamelCase						    =	MecabTokenizer(mecab_dic="""unidic_lite"""				)
								except ModuleNotFoundError:
										return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,							)

						def  A_				(			self				)					->      int:


								'''simple docstring'''

								try:
										_UpperCamelCase						    =	MecabTokenizer(mecab_dic="""unidic"""				)
								except ModuleNotFoundError:
										return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,							)

						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	MecabTokenizer(do_lower_case=a       ,							mecab_dic="""ipadic"""				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,							)

						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								try:
										_UpperCamelCase						    =	MecabTokenizer(
										    do_lower_case=a       ,							normalize_text=a       ,							mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""				)
								except RuntimeError:
										# if dict doesn't exist in the system, previous code raises this error.
										return

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,							)

						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	MecabTokenizer(normalize_text=a       ,							mecab_dic="""ipadic"""				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]       ,							)

						@require_sudachi
						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.tokenizer_class(self.vocab_file       ,							word_tokenizer_type="""sudachi"""				)
								self.assertIsNotNone(a				)

								_UpperCamelCase						    =	"""こんにちは、世界。\nこんばんは、世界。"""
								_UpperCamelCase						    =	tokenizer.tokenize(a				)
								self.assertListEqual(a       ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a				)       ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]				)

								_UpperCamelCase						    =	os.path.join(self.tmpdirname       ,							"""tokenizer.bin"""				)
								with open(a       ,							"""wb"""				) as handle:
										pickle.dump(a       ,							a				)

								with open(a       ,							"""rb"""				) as handle:
										_UpperCamelCase						    =	pickle.load(a				)

								_UpperCamelCase						    =	tokenizer_new.tokenize(a				)

								self.assertListEqual(a       ,							a				)

						@require_sudachi
						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	SudachiTokenizer(sudachi_dict_type="""core"""				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							[""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]       ,							)

						@require_sudachi
						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	SudachiTokenizer(sudachi_dict_type="""core"""       ,							sudachi_split_mode="""A"""				)

								self.assertListEqual(tokenizer.tokenize("""外国人参政権"""				)       ,							["""外国""", """人""", """参政""", """権"""]				)

						@require_sudachi
						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	SudachiTokenizer(sudachi_dict_type="""core"""       ,							sudachi_split_mode="""B"""				)

								self.assertListEqual(tokenizer.tokenize("""外国人参政権"""				)       ,							["""外国人""", """参政権"""]				)

						@require_sudachi
						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								_UpperCamelCase						    =	SudachiTokenizer(sudachi_dict_type="""core"""       ,							sudachi_split_mode="""C"""				)

								self.assertListEqual(tokenizer.tokenize("""外国人参政権"""				)       ,							["""外国人参政権"""]				)

						@require_sudachi
						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	SudachiTokenizer(do_lower_case=a       ,							sudachi_dict_type="""core"""				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							[""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]       ,							)

						@require_sudachi
						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	SudachiTokenizer(normalize_text=a       ,							sudachi_dict_type="""core"""				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							[""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]       ,							)

						@require_sudachi
						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	SudachiTokenizer(trim_whitespace=a       ,							sudachi_dict_type="""core"""				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]       ,							)

						@require_jumanpp
						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.tokenizer_class(self.vocab_file       ,							word_tokenizer_type="""jumanpp"""				)
								self.assertIsNotNone(a				)

								_UpperCamelCase						    =	"""こんにちは、世界。\nこんばんは、世界。"""
								_UpperCamelCase						    =	tokenizer.tokenize(a				)
								self.assertListEqual(a       ,							["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]				)
								self.assertListEqual(tokenizer.convert_tokens_to_ids(a				)       ,							[3, 12, 10, 14, 4, 9, 12, 10, 14]				)

								_UpperCamelCase						    =	os.path.join(self.tmpdirname       ,							"""tokenizer.bin"""				)
								with open(a       ,							"""wb"""				) as handle:
										pickle.dump(a       ,							a				)

								with open(a       ,							"""rb"""				) as handle:
										_UpperCamelCase						    =	pickle.load(a				)

								_UpperCamelCase						    =	tokenizer_new.tokenize(a				)

								self.assertListEqual(a       ,							a				)

						@require_jumanpp
						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,							)

						@require_jumanpp
						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	JumanppTokenizer(do_lower_case=a				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,							)

						@require_jumanpp
						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	JumanppTokenizer(normalize_text=a				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]       ,							)

						@require_jumanpp
						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	JumanppTokenizer(trim_whitespace=a				)

								self.assertListEqual(
								    tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """				)       ,							["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]       ,							)

						@require_jumanpp
						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	JumanppTokenizer()

								self.assertListEqual(
								    tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""				)       ,							["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]       ,							)

						def  A_				(			self				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

								_UpperCamelCase						    =	{}
								for i, token in enumerate(a				):
										_UpperCamelCase						    =	i
								_UpperCamelCase						    =	WordpieceTokenizer(vocab=a       ,							unk_token="""[UNK]"""				)

								self.assertListEqual(tokenizer.tokenize(""""""				)       ,							[]				)

								self.assertListEqual(tokenizer.tokenize("""こんにちは"""				)       ,							["""こんにちは"""]				)

								self.assertListEqual(tokenizer.tokenize("""こんばんは"""				)       ,							["""こん""", """##ばんは"""]				)

								self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""				)       ,							["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]				)

						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								_UpperCamelCase						    =	BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""				)
								_UpperCamelCase						    =	tokenizer.subword_tokenizer

								_UpperCamelCase						    =	subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""				)
								self.assertListEqual(a       ,							["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]				)

								_UpperCamelCase						    =	subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""				)
								self.assertListEqual(a       ,							["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]				)







						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""				)

								_UpperCamelCase						    =	tokenizer.encode("""ありがとう。"""       ,							add_special_tokens=a				)
								_UpperCamelCase						    =	tokenizer.encode("""どういたしまして。"""       ,							add_special_tokens=a				)

								_UpperCamelCase						    =	tokenizer.build_inputs_with_special_tokens(a				)
								_UpperCamelCase						    =	tokenizer.build_inputs_with_special_tokens(a       ,							a				)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class        lowerCAmelCase__      (   __lowercase		,	unittest.TestCase	):
						UpperCamelCase_							: Dict	=		BertJapaneseTokenizer
						UpperCamelCase_							: Optional[int]	=		False

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								super().setUp()

								_UpperCamelCase						    =	["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

								_UpperCamelCase						    =	os.path.join(self.tmpdirname       ,							VOCAB_FILES_NAMES["""vocab_file"""]				)
								with open(self.vocab_file       ,							"""w"""       ,							encoding="""utf-8"""				) as vocab_writer:
										vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]				)				)

						def  A_				(			self       ,							**a				)					->      Optional[int]:


								'''simple docstring'''

								return BertJapaneseTokenizer.from_pretrained(self.tmpdirname       ,							subword_tokenizer_type="""character"""       ,							**a				)

						def  A_				(			self       ,							a				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	"""こんにちは、世界。 \nこんばんは、世界。"""
								_UpperCamelCase						    =	"""こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
								return input_text, output_text

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								pass  # TODO add if relevant

						def  A_				(			self				)					->      Optional[Any]:


								'''simple docstring'''

								pass  # TODO add if relevant

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								pass  # TODO add if relevant

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.tokenizer_class(self.vocab_file       ,							subword_tokenizer_type="""character"""				)

								_UpperCamelCase						    =	tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""				)
								self.assertListEqual(
								    a       ,							["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]				)
								self.assertListEqual(
								    tokenizer.convert_tokens_to_ids(a				)       ,							[3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]				)

						def  A_				(			self				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

								_UpperCamelCase						    =	{}
								for i, token in enumerate(a				):
										_UpperCamelCase						    =	i
								_UpperCamelCase						    =	CharacterTokenizer(vocab=a       ,							unk_token="""[UNK]"""				)

								self.assertListEqual(tokenizer.tokenize(""""""				)       ,							[]				)

								self.assertListEqual(tokenizer.tokenize("""こんにちは"""				)       ,							["""こ""", """ん""", """に""", """ち""", """は"""]				)

								self.assertListEqual(tokenizer.tokenize("""こんにちほ"""				)       ,							["""こ""", """ん""", """に""", """ち""", """[UNK]"""]				)







						def  A_				(			self				)					->      Union[str, Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""				)

								_UpperCamelCase						    =	tokenizer.encode("""ありがとう。"""       ,							add_special_tokens=a				)
								_UpperCamelCase						    =	tokenizer.encode("""どういたしまして。"""       ,							add_special_tokens=a				)

								_UpperCamelCase						    =	tokenizer.build_inputs_with_special_tokens(a				)
								_UpperCamelCase						    =	tokenizer.build_inputs_with_special_tokens(a       ,							a				)

								# 2 is for "[CLS]", 3 is for "[SEP]"
								assert encoded_sentence == [2] + text + [3]
								assert encoded_pair == [2] + text + [3] + text_a + [3]







@custom_tokenizers
class        lowerCAmelCase__      (   unittest.TestCase	):

						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	"""cl-tohoku/bert-base-japanese"""
								_UpperCamelCase						    =	AutoTokenizer.from_pretrained(a				)
								self.assertIsInstance(a       ,							a				)







class        lowerCAmelCase__      (   unittest.TestCase	):

						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	"""cl-tohoku/bert-base-japanese"""
								with self.assertLogs("""transformers"""       ,							level="""WARNING"""				) as cm:
										BertTokenizer.from_pretrained(a				)
										self.assertTrue(
										    cm.records[0].message.startswith(
										        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
										        """ is called from."""				)				)
								_UpperCamelCase						    =	"""bert-base-cased"""
								with self.assertLogs("""transformers"""       ,							level="""WARNING"""				) as cm:
										BertJapaneseTokenizer.from_pretrained(a				)
										self.assertTrue(
										    cm.records[0].message.startswith(
										        """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
										        """ is called from."""				)				)





from typing import TYPE_CHECKING

from ....utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowerCamelCase__	=							{
    "configuration_mctct": ["MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MCTCTConfig"],
    "feature_extraction_mctct": ["MCTCTFeatureExtractor"],
    "processing_mctct": ["MCTCTProcessor"],
}


try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowerCamelCase__	=							[
		    "MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST",
		    "MCTCTForCTC",
		    "MCTCTModel",
		    "MCTCTPreTrainedModel",
		]


if TYPE_CHECKING:
		from .configuration_mctct import MCTCT_PRETRAINED_CONFIG_ARCHIVE_MAP, MCTCTConfig
		from .feature_extraction_mctct import MCTCTFeatureExtractor
		from .processing_mctct import MCTCTProcessor

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_mctct import MCTCT_PRETRAINED_MODEL_ARCHIVE_LIST, MCTCTForCTC, MCTCTModel, MCTCTPreTrainedModel

else:
		import sys

		lowerCamelCase__	=							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



