





'''simple docstring'''




import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin
@custom_tokenizers
class 			_lowerCAmelCase		(			A__							,     unittest.TestCase	):






     """simple docstring"""






     snake_case_						      =   BertJapaneseTokenizer
     snake_case_						      =   False
     snake_case_						      =   True



     def      lowerCAmelCase			(   self							:			Any						)->	str:
        super().setUp()

        snake_case 						=   [
            """[UNK]""",
            """[CLS]""",
            """[SEP]""",
            """こんにちは""",
            """こん""",
            """にちは""",
            """ばんは""",
            """##こん""",
            """##にちは""",
            """##ばんは""",
            """世界""",
            """##世界""",
            """、""",
            """##、""",
            """。""",
            """##。""",
        ]

        snake_case 						=   os.path.join(self.tmpdirname   ,       VOCAB_FILES_NAMES["""vocab_file"""]						)
        with open(self.vocab_file   ,       """w"""   ,       encoding="""utf-8"""						) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)



     def      lowerCAmelCase			(   self							:			Dict   ,       __snake_case							:			Optional[int]						)->	Union[str, Any]:
        snake_case 						=   """こんにちは、世界。 \nこんばんは、世界。"""
        snake_case 						=   """こんにちは 、 世界 。 こんばんは 、 世界 。"""
        return input_text, output_text



     def      lowerCAmelCase			(   self							:			Optional[int]   ,       __snake_case							:			str						)->	Dict:
        snake_case						,    snake_case 						=   self.get_input_output_texts(__snake_case						)
        snake_case 						=   tokenizer.encode(__snake_case   ,       add_special_tokens=__snake_case						)
        snake_case 						=   tokenizer.decode(__snake_case   ,       clean_up_tokenization_spaces=__snake_case						)
        return text, ids



     def      lowerCAmelCase			(   self							:			List[Any]						)->	List[str]:
        pass  # TODO add if relevant



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Optional[int]:
        pass  # TODO add if relevant



     def      lowerCAmelCase			(   self							:			Tuple						)->	Union[str, Any]:
        pass  # TODO add if relevant



     def      lowerCAmelCase			(   self							:			Optional[int]						)->	int:
        snake_case 						=   self.tokenizer_class(self.vocab_file						)

        snake_case 						=   tokenizer.tokenize("""こんにちは、世界。\nこんばんは、世界。"""						)
        self.assertListEqual(__snake_case   ,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case						)   ,       [3, 12, 10, 14, 4, 9, 12, 10, 14]						)



     def      lowerCAmelCase			(   self							:			Optional[int]						)->	str:
        snake_case 						=   self.tokenizer_class(self.vocab_file   ,       word_tokenizer_type="""mecab"""						)
        self.assertIsNotNone(__snake_case						)

        snake_case 						=   """こんにちは、世界。\nこんばんは、世界。"""
        snake_case 						=   tokenizer.tokenize(__snake_case						)
        self.assertListEqual(__snake_case   ,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case						)   ,       [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

        snake_case 						=   os.path.join(self.tmpdirname   ,       """tokenizer.bin"""						)
        with open(__snake_case   ,       """wb"""						) as handle:
           pickle.dump(__snake_case   ,       __snake_case						)

        with open(__snake_case   ,       """rb"""						) as handle:
           snake_case 						=   pickle.load(__snake_case						)

        snake_case 						=   tokenizer_new.tokenize(__snake_case						)

        self.assertListEqual(__snake_case   ,       __snake_case						)



     def      lowerCAmelCase			(   self							:			List[Any]						)->	str:
        snake_case 						=   MecabTokenizer(mecab_dic="""ipadic"""						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップルストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,       )



     def      lowerCAmelCase			(   self							:			List[str]						)->	List[str]:
        try:
           snake_case 						=   MecabTokenizer(mecab_dic="""unidic_lite"""						)
        except ModuleNotFoundError:
           return

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,       )



     def      lowerCAmelCase			(   self							:			int						)->	List[str]:
        try:
           snake_case 						=   MecabTokenizer(mecab_dic="""unidic"""						)
        except ModuleNotFoundError:
           return

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,       )



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Optional[Any]:
        snake_case 						=   MecabTokenizer(do_lower_case=__snake_case   ,       mecab_dic="""ipadic"""						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップルストア""", """で""", """iphone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,       )



     def      lowerCAmelCase			(   self							:			Dict						)->	str:
        try:
           snake_case 						=   MecabTokenizer(
               do_lower_case=__snake_case   ,       normalize_text=__snake_case   ,       mecab_option="""-d /usr/local/lib/mecab/dic/jumandic"""						)
        except RuntimeError:
           # if dict doesn't exist in the system, previous code raises this error.
           return

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,       )



     def      lowerCAmelCase			(   self							:			Dict						)->	Tuple:
        snake_case 						=   MecabTokenizer(normalize_text=__snake_case   ,       mecab_dic="""ipadic"""						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""ｱｯﾌﾟﾙストア""", """で""", """iPhone""", """８""", """が""", """発売""", """さ""", """れ""", """た""", """　""", """。"""]   ,       )



     @require_sudachi
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	List[str]:
        snake_case 						=   self.tokenizer_class(self.vocab_file   ,       word_tokenizer_type="""sudachi"""						)
        self.assertIsNotNone(__snake_case						)

        snake_case 						=   """こんにちは、世界。\nこんばんは、世界。"""
        snake_case 						=   tokenizer.tokenize(__snake_case						)
        self.assertListEqual(__snake_case   ,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case						)   ,       [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

        snake_case 						=   os.path.join(self.tmpdirname   ,       """tokenizer.bin"""						)
        with open(__snake_case   ,       """wb"""						) as handle:
           pickle.dump(__snake_case   ,       __snake_case						)

        with open(__snake_case   ,       """rb"""						) as handle:
           snake_case 						=   pickle.load(__snake_case						)

        snake_case 						=   tokenizer_new.tokenize(__snake_case						)

        self.assertListEqual(__snake_case   ,       __snake_case						)



     @require_sudachi
     def      lowerCAmelCase			(   self							:			List[str]						)->	Dict:
        snake_case 						=   SudachiTokenizer(sudachi_dict_type="""core"""						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       [""" """, """\t""", """アップル""", """ストア""", """で""", """iPhone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]   ,       )



     @require_sudachi
     def      lowerCAmelCase			(   self							:			List[Any]						)->	List[str]:
        snake_case 						=   SudachiTokenizer(sudachi_dict_type="""core"""   ,       sudachi_split_mode="""A"""						)

        self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)   ,       ["""外国""", """人""", """参政""", """権"""]						)



     @require_sudachi
     def      lowerCAmelCase			(   self							:			Union[str, Any]						)->	Optional[Any]:
        snake_case 						=   SudachiTokenizer(sudachi_dict_type="""core"""   ,       sudachi_split_mode="""B"""						)

        self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)   ,       ["""外国人""", """参政権"""]						)



     @require_sudachi
     def      lowerCAmelCase			(   self							:			List[str]						)->	Optional[Any]:
        snake_case 						=   SudachiTokenizer(sudachi_dict_type="""core"""   ,       sudachi_split_mode="""C"""						)

        self.assertListEqual(tokenizer.tokenize("""外国人参政権"""						)   ,       ["""外国人参政権"""]						)



     @require_sudachi
     def      lowerCAmelCase			(   self							:			List[Any]						)->	int:
        snake_case 						=   SudachiTokenizer(do_lower_case=__snake_case   ,       sudachi_dict_type="""core"""						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       [""" """, """\t""", """アップル""", """ストア""", """で""", """iphone""", """8""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """ """, """。""", """ """, """ """]   ,       )



     @require_sudachi
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Dict:
        snake_case 						=   SudachiTokenizer(normalize_text=__snake_case   ,       sudachi_dict_type="""core"""						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       [""" """, """\t""", """ｱｯﾌﾟﾙ""", """ストア""", """で""", """iPhone""", """８""", """ """, """が""", """ """, """ """, """\n """, """発売""", """さ""", """れ""", """た""", """\u3000""", """。""", """ """, """ """]   ,       )



     @require_sudachi
     def      lowerCAmelCase			(   self							:			Optional[int]						)->	Optional[Any]:
        snake_case 						=   SudachiTokenizer(trim_whitespace=__snake_case   ,       sudachi_dict_type="""core"""						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れ""", """た""", """。"""]   ,       )



     @require_jumanpp
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	int:
        snake_case 						=   self.tokenizer_class(self.vocab_file   ,       word_tokenizer_type="""jumanpp"""						)
        self.assertIsNotNone(__snake_case						)

        snake_case 						=   """こんにちは、世界。\nこんばんは、世界。"""
        snake_case 						=   tokenizer.tokenize(__snake_case						)
        self.assertListEqual(__snake_case   ,       ["""こんにちは""", """、""", """世界""", """。""", """こん""", """##ばんは""", """、""", """世界""", """。"""]						)
        self.assertListEqual(tokenizer.convert_tokens_to_ids(__snake_case						)   ,       [3, 12, 10, 14, 4, 9, 12, 10, 14]						)

        snake_case 						=   os.path.join(self.tmpdirname   ,       """tokenizer.bin"""						)
        with open(__snake_case   ,       """wb"""						) as handle:
           pickle.dump(__snake_case   ,       __snake_case						)

        with open(__snake_case   ,       """rb"""						) as handle:
           snake_case 						=   pickle.load(__snake_case						)

        snake_case 						=   tokenizer_new.tokenize(__snake_case						)

        self.assertListEqual(__snake_case   ,       __snake_case						)



     @require_jumanpp
     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Union[str, Any]:
        snake_case 						=   JumanppTokenizer()

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,       )



     @require_jumanpp
     def      lowerCAmelCase			(   self							:			Dict						)->	Optional[int]:
        snake_case 						=   JumanppTokenizer(do_lower_case=__snake_case						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップル""", """ストア""", """で""", """iphone""", """8""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,       )



     @require_jumanpp
     def      lowerCAmelCase			(   self							:			List[str]						)->	Union[str, Any]:
        snake_case 						=   JumanppTokenizer(normalize_text=__snake_case						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""ｱ""", """ｯ""", """ﾌ""", """ﾟ""", """ﾙ""", """ストア""", """で""", """iPhone""", """８""", """\u3000""", """が""", """\u3000""", """\u3000""", """\u3000""", """発売""", """さ""", """れた""", """\u3000""", """。"""]   ,       )



     @require_jumanpp
     def      lowerCAmelCase			(   self							:			List[str]						)->	List[Any]:
        snake_case 						=   JumanppTokenizer(trim_whitespace=__snake_case						)

        self.assertListEqual(
            tokenizer.tokenize(""" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  """						)   ,       ["""アップル""", """ストア""", """で""", """iPhone""", """8""", """が""", """発売""", """さ""", """れた""", """。"""]   ,       )



     @require_jumanpp
     def      lowerCAmelCase			(   self							:			Any						)->	Any:
        snake_case 						=   JumanppTokenizer()

        self.assertListEqual(
            tokenizer.tokenize("""ありがとうございますm(_ _)ｍ見つけるのが大変です。"""						)   ,       ["""ありがとう""", """ございます""", """m(_ _)m""", """見つける""", """の""", """が""", """大変です""", """。"""]   ,       )



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	int:
        snake_case 						=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こんにちは""", """こん""", """にちは""", """ばんは""", """##こん""", """##にちは""", """##ばんは"""]

        snake_case 						=   {}
        for i, token in enumerate(__snake_case						):
           snake_case 						=   i
        snake_case 						=   WordpieceTokenizer(vocab=__snake_case   ,       unk_token="""[UNK]"""						)

        self.assertListEqual(tokenizer.tokenize(""""""						)   ,       []						)

        self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)   ,       ["""こんにちは"""]						)

        self.assertListEqual(tokenizer.tokenize("""こんばんは"""						)   ,       ["""こん""", """##ばんは"""]						)

        self.assertListEqual(tokenizer.tokenize("""こんばんは こんばんにちは こんにちは"""						)   ,       ["""こん""", """##ばんは""", """[UNK]""", """こんにちは"""]						)



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Tuple:
        snake_case 						=   BertJapaneseTokenizer.from_pretrained("""nlp-waseda/roberta-base-japanese-with-auto-jumanpp"""						)
        snake_case 						=   tokenizer.subword_tokenizer

        snake_case 						=   subword_tokenizer.tokenize("""国境 の 長い トンネル を 抜ける と 雪国 であった 。"""						)
        self.assertListEqual(__snake_case   ,       ["""▁国境""", """▁の""", """▁長い""", """▁トンネル""", """▁を""", """▁抜ける""", """▁と""", """▁雪""", """国""", """▁であった""", """▁。"""]						)

        snake_case 						=   subword_tokenizer.tokenize("""こんばんは こんばん にち は こんにちは"""						)
        self.assertListEqual(__snake_case   ,       ["""▁こん""", """ばん""", """は""", """▁こん""", """ばん""", """▁に""", """ち""", """▁は""", """▁こんにちは"""]						)



     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Optional[int]:
        snake_case 						=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese"""						)

        snake_case 						=   tokenizer.encode("""ありがとう。"""   ,       add_special_tokens=__snake_case						)
        snake_case 						=   tokenizer.encode("""どういたしまして。"""   ,       add_special_tokens=__snake_case						)

        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case						)
        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case   ,       __snake_case						)

        # 2 is for "[CLS]", 3 is for "[SEP]"
        assert encoded_sentence == [2] + text + [3]
        assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			_lowerCAmelCase		(			A__							,     unittest.TestCase	):






     """simple docstring"""






     snake_case_						      =   BertJapaneseTokenizer
     snake_case_						      =   False



     def      lowerCAmelCase			(   self							:			Tuple						)->	Any:
        super().setUp()

        snake_case 						=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

        snake_case 						=   os.path.join(self.tmpdirname   ,       VOCAB_FILES_NAMES["""vocab_file"""]						)
        with open(self.vocab_file   ,       """w"""   ,       encoding="""utf-8"""						) as vocab_writer:
           vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]						)						)



     def      lowerCAmelCase			(   self							:			Dict   ,       **__snake_case							:			int						)->	Union[str, Any]:
        return BertJapaneseTokenizer.from_pretrained(self.tmpdirname   ,       subword_tokenizer_type="""character"""   ,       **__snake_case						)



     def      lowerCAmelCase			(   self							:			List[Any]   ,       __snake_case							:			Optional[int]						)->	Any:
        snake_case 						=   """こんにちは、世界。 \nこんばんは、世界。"""
        snake_case 						=   """こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"""
        return input_text, output_text



     def      lowerCAmelCase			(   self							:			Dict						)->	Optional[Any]:
        pass  # TODO add if relevant



     def      lowerCAmelCase			(   self							:			Tuple						)->	Union[str, Any]:
        pass  # TODO add if relevant



     def      lowerCAmelCase			(   self							:			List[str]						)->	Any:
        pass  # TODO add if relevant



     def      lowerCAmelCase			(   self							:			Tuple						)->	int:
        snake_case 						=   self.tokenizer_class(self.vocab_file   ,       subword_tokenizer_type="""character"""						)

        snake_case 						=   tokenizer.tokenize("""こんにちは、世界。 \nこんばんは、世界。"""						)
        self.assertListEqual(
            __snake_case   ,       ["""こ""", """ん""", """に""", """ち""", """は""", """、""", """世""", """界""", """。""", """こ""", """ん""", """ば""", """ん""", """は""", """、""", """世""", """界""", """。"""]						)
        self.assertListEqual(
            tokenizer.convert_tokens_to_ids(__snake_case						)   ,       [3, 4, 5, 6, 7, 11, 9, 10, 12, 3, 4, 8, 4, 7, 11, 9, 10, 12]						)



     def      lowerCAmelCase			(   self							:			Dict						)->	Dict:
        snake_case 						=   ["""[UNK]""", """[CLS]""", """[SEP]""", """こ""", """ん""", """に""", """ち""", """は""", """ば""", """世""", """界""", """、""", """。"""]

        snake_case 						=   {}
        for i, token in enumerate(__snake_case						):
           snake_case 						=   i
        snake_case 						=   CharacterTokenizer(vocab=__snake_case   ,       unk_token="""[UNK]"""						)

        self.assertListEqual(tokenizer.tokenize(""""""						)   ,       []						)

        self.assertListEqual(tokenizer.tokenize("""こんにちは"""						)   ,       ["""こ""", """ん""", """に""", """ち""", """は"""]						)

        self.assertListEqual(tokenizer.tokenize("""こんにちほ"""						)   ,       ["""こ""", """ん""", """に""", """ち""", """[UNK]"""]						)



     def      lowerCAmelCase			(   self							:			Optional[int]						)->	List[Any]:
        snake_case 						=   self.tokenizer_class.from_pretrained("""cl-tohoku/bert-base-japanese-char"""						)

        snake_case 						=   tokenizer.encode("""ありがとう。"""   ,       add_special_tokens=__snake_case						)
        snake_case 						=   tokenizer.encode("""どういたしまして。"""   ,       add_special_tokens=__snake_case						)

        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case						)
        snake_case 						=   tokenizer.build_inputs_with_special_tokens(__snake_case   ,       __snake_case						)

        # 2 is for "[CLS]", 3 is for "[SEP]"
        assert encoded_sentence == [2] + text + [3]
        assert encoded_pair == [2] + text + [3] + text_a + [3]




@custom_tokenizers
class 			_lowerCAmelCase		(			unittest.TestCase	):






     """simple docstring"""






     def      lowerCAmelCase			(   self							:			Tuple						)->	Any:
        snake_case 						=   """cl-tohoku/bert-base-japanese"""
        snake_case 						=   AutoTokenizer.from_pretrained(__snake_case						)
        self.assertIsInstance(__snake_case   ,       __snake_case						)




class 			_lowerCAmelCase		(			unittest.TestCase	):






     """simple docstring"""






     def      lowerCAmelCase			(   self							:			str						)->	Tuple:
        snake_case 						=   """cl-tohoku/bert-base-japanese"""
        with self.assertLogs("""transformers"""   ,       level="""WARNING"""						) as cm:
           BertTokenizer.from_pretrained(__snake_case						)
           self.assertTrue(
               cm.records[0].message.startswith(
                   """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                   """ is called from."""						)						)
        snake_case 						=   """bert-base-cased"""
        with self.assertLogs("""transformers"""   ,       level="""WARNING"""						) as cm:
           BertJapaneseTokenizer.from_pretrained(__snake_case						)
           self.assertTrue(
               cm.records[0].message.startswith(
                   """The tokenizer class you load from this checkpoint is not the same type as the class this function"""
                   """ is called from."""						)						)








'''simple docstring'''




from manim import *
class 			_lowerCAmelCase		(			A__	):






     """simple docstring"""






     def      lowerCAmelCase			(   self							:			Optional[Any]						)->	Union[str, Any]:
        snake_case 						=   Rectangle(height=0.5   ,       width=0.5						)
        snake_case 						=   Rectangle(height=0.25   ,       width=0.25						)
        snake_case 						=   Rectangle(height=0.46   ,       width=0.46						).set_stroke(width=0						)

        snake_case 						=   [mem.copy() for i in range(6						)]
        snake_case 						=   [mem.copy() for i in range(6						)]
        snake_case 						=   VGroup(*__snake_case						).arrange(__snake_case   ,       buff=0						)
        snake_case 						=   VGroup(*__snake_case						).arrange(__snake_case   ,       buff=0						)
        snake_case 						=   VGroup(__snake_case   ,       __snake_case						).arrange(__snake_case   ,       buff=0						)
        snake_case 						=   Text("""CPU"""   ,       font_size=24						)
        snake_case 						=   Group(__snake_case   ,       __snake_case						).arrange(__snake_case   ,       buff=0.5   ,       aligned_edge=__snake_case						)
        cpu.move_to([-2.5, -0.5, 0]						)
        self.add(__snake_case						)

        snake_case 						=   [mem.copy() for i in range(4						)]
        snake_case 						=   VGroup(*__snake_case						).arrange(__snake_case   ,       buff=0						)
        snake_case 						=   Text("""GPU"""   ,       font_size=24						)
        snake_case 						=   Group(__snake_case   ,       __snake_case						).arrange(__snake_case   ,       buff=0.5   ,       aligned_edge=__snake_case						)
        gpu.move_to([-1, -1, 0]						)
        self.add(__snake_case						)

        snake_case 						=   [mem.copy() for i in range(6						)]
        snake_case 						=   VGroup(*__snake_case						).arrange(__snake_case   ,       buff=0						)

        snake_case 						=   Text("""Model"""   ,       font_size=24						)
        snake_case 						=   Group(__snake_case   ,       __snake_case						).arrange(__snake_case   ,       buff=0.5   ,       aligned_edge=__snake_case						)
        model.move_to([3, -1.0, 0]						)
        self.add(__snake_case						)

        snake_case 						=   []
        snake_case 						=   []
        snake_case 						=   []

        for i, rect in enumerate(__snake_case						):
           rect.set_stroke(__snake_case						)

           snake_case 						=   Rectangle(height=0.46 / 4   ,       width=0.46 / 3						).set_stroke(width=0.0						).set_fill(__snake_case   ,       opacity=0.7						)

           if i == 0:
              cpu_target.next_to(cpu_left_col_base[0].get_corner(DOWN + LEFT						)   ,       buff=0.02   ,       direction=__snake_case						)
              cpu_target.set_x(cpu_target.get_x() + 0.1						)
           elif i == 3:
              cpu_target.next_to(model_cpu_arr[0]   ,       direction=__snake_case   ,       buff=0.0						)
           else:
              cpu_target.next_to(model_cpu_arr[i - 1]   ,       direction=__snake_case   ,       buff=0.0						)
           self.add(__snake_case						)
           model_cpu_arr.append(__snake_case						)

        self.add(*__snake_case   ,       *__snake_case   ,       *__snake_case						)

        snake_case 						=   [mem.copy() for i in range(6						)]
        snake_case 						=   VGroup(*__snake_case						).arrange(__snake_case   ,       buff=0						)

        snake_case 						=   Text("""Loaded Checkpoint"""   ,       font_size=24						)
        snake_case 						=   Group(__snake_case   ,       __snake_case						).arrange(__snake_case   ,       buff=0.5   ,       aligned_edge=__snake_case						)
        checkpoint.move_to([3, 0.5, 0]						)

        self.add(__snake_case						)

        snake_case 						=   []
        snake_case 						=   []

        for i, rect in enumerate(__snake_case						):
           snake_case 						=   fill.copy().set_fill(__snake_case   ,       opacity=0.7						)
           target.move_to(__snake_case						)
           ckpt_arr.append(__snake_case						)

           snake_case 						=   target.copy()
           if i < 5:
              cpu_target.move_to(cpu_left_col_base[i + 1]						)
           else:
              cpu_target.move_to(cpu_right_col_base[i - 5]						)
           ckpt_cpu_arr.append(__snake_case						)
        self.add(*__snake_case   ,       *__snake_case						)

        snake_case 						=   Square(side_length=2.2						)
        key.move_to([-5, 2, 0]						)

        snake_case 						=   MarkupText(
            f'''<b>Key:</b>\n\n<span fgcolor=\'{YELLOW}\'>●</span> Empty Model'''   ,       font_size=18   ,       )

        key_text.move_to([-5, 2.4, 0]						)

        self.add(__snake_case   ,       __snake_case						)

        snake_case 						=   MarkupText(
            f'''<span fgcolor=\'{BLUE}\'>●</span> Checkpoint'''   ,       font_size=18   ,       )

        blue_text.next_to(__snake_case   ,       DOWN * 2.4   ,       aligned_edge=key_text.get_left()						)
        self.add(__snake_case						)

        snake_case 						=   MarkupText(
            f'''Based on the passed in configuration, weights are stored in\na variety of np.memmaps on disk or to a particular device.'''   ,       font_size=24   ,       )
        step_a.move_to([2, 2, 0]						)

        snake_case 						=   [meta_mem.copy() for i in range(6						)]
        snake_case 						=   [meta_mem.copy() for i in range(6						)]
        snake_case 						=   VGroup(*__snake_case						).arrange(__snake_case   ,       buff=0						)
        snake_case 						=   VGroup(*__snake_case						).arrange(__snake_case   ,       buff=0						)
        snake_case 						=   VGroup(__snake_case   ,       __snake_case						).arrange(__snake_case   ,       buff=0						)
        snake_case 						=   Text("""Disk"""   ,       font_size=24						)
        snake_case 						=   Group(__snake_case   ,       __snake_case						).arrange(__snake_case   ,       buff=0.5   ,       aligned_edge=__snake_case						)
        disk.move_to([-4.0, -1.25, 0]						)
        self.play(Write(__snake_case   ,       run_time=3						)   ,       Write(__snake_case   ,       run_time=1						)   ,       Create(__snake_case   ,       run_time=1						)						)

        snake_case 						=   []
        for i, rect in enumerate(__snake_case						):
           snake_case 						=   rect.copy()
           target.generate_target()
           target.target.move_to(disk_left_col_base[i]						).scale(0.5						)
           animations.append(MoveToTarget(__snake_case   ,       run_time=1.5						)						)
        self.play(*__snake_case						)

        self.play(FadeOut(__snake_case						)						)

        snake_case 						=   MarkupText(f'''Then, the checkpoint is removed from memory\nthrough garbage collection.'''   ,       font_size=24						)
        step_a.move_to([2, 2, 0]						)

        self.play(Write(__snake_case   ,       run_time=3						)						)

        self.play(
            FadeOut(__snake_case   ,       __snake_case   ,       *__snake_case   ,       *__snake_case						)   ,       )

        self.wait()

