





import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, PerceiverTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
       UpperCamelCase__      ='pt'
elif is_tf_available():
       UpperCamelCase__      ='tf'
else:
       UpperCamelCase__      ='jax'





class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    PerceiverTokenizer
       __snake_case 			=    False




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            super().setUp()
            _SCREAMING_SNAKE_CASE       :	int						    =							PerceiverTokenizer()
            tokenizer.save_pretrained(self.tmpdirname							)




       @cached_property
       def     UpperCamelCase_			(      self							)			->     List[str]:
            return PerceiverTokenizer.from_pretrained("deepmind/language-perceiver"							)




       def     UpperCamelCase_			(      self    ,						**__lowerCamelCase							)			->     Optional[int]:
            return self.tokenizer_class.from_pretrained(self.tmpdirname    ,						**A__							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase    ,						__lowerCamelCase=False    ,						__lowerCamelCase=2_0    ,						__lowerCamelCase=5							)			->     List[Any]:
            # XXX The default common tokenizer tests assume that every ID is decodable on its own.
            # This assumption is invalid for Perceiver because single bytes might not be
            # valid utf-8 (byte 128 for instance).
            # Here we're overriding the smallest possible method to provide
            # a clean sequence without making the same assumption.

            _SCREAMING_SNAKE_CASE       :	int						    =							[]
            for i in range(len(A__							)							):
                 try:
                      _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.decode([i]    ,						clean_up_tokenization_spaces=A__							)
                 except UnicodeDecodeError:
                      pass
                 toks.append((i, tok)							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							list(filter(lambda __lowerCamelCase							: re.match(r"^[ a-zA-Z]+$"    ,						t[1]							)    ,						A__							)							)
            _SCREAMING_SNAKE_CASE       :	str						    =							list(filter(lambda __lowerCamelCase							: [t[0]] == tokenizer.encode(t[1]    ,						add_special_tokens=A__							)    ,						A__							)							)
            if max_length is not None and len(A__							) > max_length:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							toks[:max_length]
            if min_length is not None and len(A__							) < min_length and len(A__							) > 0:
                 while len(A__							) < min_length:
                      _SCREAMING_SNAKE_CASE       :	List[Any]						    =							toks + toks
        # toks_str = [t[1] for t in toks]
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							[t[0] for t in toks]

            # Ensure consistency
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenizer.decode(A__    ,						clean_up_tokenization_spaces=A__							)
            if " " not in output_txt and len(A__							) > 1:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							(
                     tokenizer.decode([toks_ids[0]]    ,						clean_up_tokenization_spaces=A__							)
                     + """ """
                     + tokenizer.decode(toks_ids[1:]    ,						clean_up_tokenization_spaces=A__							)
                 )
            if with_prefix_space:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							""" """ + output_txt
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tokenizer.encode(A__    ,						add_special_tokens=A__							)
            return output_txt, output_ids




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.perceiver_tokenizer
            _SCREAMING_SNAKE_CASE       :	str						    =							"""Unicode €."""
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenizer(A__							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							[4, 9_1, 1_1_6, 1_1_1, 1_0_5, 1_1_7, 1_0_6, 1_0_7, 3_8, 2_3_2, 1_3_6, 1_7_8, 5_2, 5]
            self.assertEqual(encoded["input_ids"]    ,						A__							)

            # decoding
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.decode(A__							)
            self.assertEqual(A__    ,						"[CLS]Unicode €.[SEP]"							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer("e è é ê ë"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							[4, 1_0_7, 3_8, 2_0_1, 1_7_4, 3_8, 2_0_1, 1_7_5, 3_8, 2_0_1, 1_7_6, 3_8, 2_0_1, 1_7_7, 5]
            self.assertEqual(encoded["input_ids"]    ,						A__							)
            # decoding
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							tokenizer.decode(A__							)
            self.assertEqual(A__    ,						"[CLS]e è é ê ë[SEP]"							)

            # encode/decode, but with `encode` instead of `__call__`
            self.assertEqual(tokenizer.decode(tokenizer.encode("e è é ê ë"							)							)    ,						"[CLS]e è é ê ë[SEP]"							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	int						    =							self.perceiver_tokenizer
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
            # fmt: off
            _SCREAMING_SNAKE_CASE       :	str						    =							[4, 7_1, 3_8, 1_1_4, 1_1_7, 1_1_6, 1_0_9, 3_8, 1_1_8, 1_0_3, 1_2_0, 1_0_3, 1_0_9, 1_2_0, 1_0_3, 1_1_8, 1_1_0, 3_8, 1_0_8, 1_1_7, 1_2_0, 3_8, 1_2_1, 1_2_3, 1_1_5, 1_1_5, 1_0_3, 1_2_0, 1_1_1, 1_2_8, 1_0_3, 1_2_2, 1_1_1, 1_1_7, 1_1_6, 5_2, 5, 0]
            # fmt: on
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							tokenizer(A__    ,						padding=A__    ,						return_tensors=A__							)
            self.assertIsInstance(A__    ,						A__							)

            if FRAMEWORK != "jax":
                 _SCREAMING_SNAKE_CASE       :	Dict						    =							list(batch.input_ids.numpy()[0]							)
            else:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							list(batch.input_ids.tolist()[0]							)

            self.assertListEqual(A__    ,						A__							)

            self.assertEqual((2, 3_8)    ,						batch.input_ids.shape							)
            self.assertEqual((2, 3_8)    ,						batch.attention_mask.shape							)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.perceiver_tokenizer
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							["""A long paragraph for summarization.""", """Another paragraph for summarization."""]
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							tokenizer(A__    ,						padding=A__    ,						return_tensors=A__							)
            # check if input_ids are returned and no decoder_input_ids
            self.assertIn("input_ids"    ,						A__							)
            self.assertIn("attention_mask"    ,						A__							)
            self.assertNotIn("decoder_input_ids"    ,						A__							)
            self.assertNotIn("decoder_attention_mask"    ,						A__							)




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.perceiver_tokenizer
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							[
                """Summary of the text.""",
                """Another summary.""",
            ]
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tokenizer(
                text_target=A__    ,						max_length=3_2    ,						padding="max_length"    ,						truncation=A__    ,						return_tensors=A__							)
            self.assertEqual(3_2    ,						targets["input_ids"].shape[1]							)




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            # safety check on max_len default value so we are sure the test works
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							self.get_tokenizers()
            for tokenizer in tokenizers:
                 with self.subTest(F"""{tokenizer.__class__.__name__}"""							):
                      self.assertNotEqual(tokenizer.model_max_length    ,						4_2							)

        # Now let's start the test
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.get_tokenizers()
            for tokenizer in tokenizers:
                 with self.subTest(F"""{tokenizer.__class__.__name__}"""							):
                      # Isolate this from the other tests because we save additional tokens/etc
                      _SCREAMING_SNAKE_CASE       :	Tuple						    =							tempfile.mkdtemp()

                      _SCREAMING_SNAKE_CASE       :	Tuple						    =							""" He is very happy, UNwant\u00E9d,running"""
                      _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tokenizer.encode(A__    ,						add_special_tokens=A__							)
                      tokenizer.save_pretrained(A__							)

                      _SCREAMING_SNAKE_CASE       :	List[str]						    =							tokenizer.__class__.from_pretrained(A__							)
                      _SCREAMING_SNAKE_CASE       :	int						    =							after_tokenizer.encode(A__    ,						add_special_tokens=A__							)
                      self.assertListEqual(A__    ,						A__							)

                      shutil.rmtree(A__							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							self.get_tokenizers(model_max_length=4_2							)
            for tokenizer in tokenizers:
                 with self.subTest(F"""{tokenizer.__class__.__name__}"""							):
                      # Isolate this from the other tests because we save additional tokens/etc
                      _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tempfile.mkdtemp()

                      _SCREAMING_SNAKE_CASE       :	Any						    =							""" He is very happy, UNwant\u00E9d,running"""
                      tokenizer.add_tokens(["bim", "bambam"]							)
                      _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tokenizer.additional_special_tokens
                      additional_special_tokens.append("new_additional_special_token"							)
                      tokenizer.add_special_tokens({"additional_special_tokens": additional_special_tokens}							)
                      _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							tokenizer.encode(A__    ,						add_special_tokens=A__							)
                      tokenizer.save_pretrained(A__							)

                      _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.__class__.from_pretrained(A__							)
                      _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							after_tokenizer.encode(A__    ,						add_special_tokens=A__							)
                      self.assertListEqual(A__    ,						A__							)
                      self.assertIn("new_additional_special_token"    ,						after_tokenizer.additional_special_tokens							)
                      self.assertEqual(after_tokenizer.model_max_length    ,						4_2							)

                      _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.__class__.from_pretrained(A__    ,						model_max_length=4_3							)
                      self.assertEqual(tokenizer.model_max_length    ,						4_3							)

                      shutil.rmtree(A__							)




       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							[]
            if self.test_slow_tokenizer:
                 tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())							)

            if self.test_rust_tokenizer:
                 tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())							)

            for tokenizer_class, tokenizer_utils in tokenizer_list:
                 with tempfile.TemporaryDirectory() as tmp_dir:
                      tokenizer_utils.save_pretrained(A__							)

                      with open(os.path.join(A__    ,						"special_tokens_map.json"							)    ,						encoding="utf-8"							) as json_file:
                           _SCREAMING_SNAKE_CASE       :	int						    =							json.load(A__							)

                      with open(os.path.join(A__    ,						"tokenizer_config.json"							)    ,						encoding="utf-8"							) as json_file:
                           _SCREAMING_SNAKE_CASE       :	Tuple						    =							json.load(A__							)

                      _SCREAMING_SNAKE_CASE       :	str						    =							[F"""<extra_id_{i}>""" for i in range(1_2_5							)]

                      _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							added_tokens_extra_ids + [
                          """an_additional_special_token"""
                      ]
                      _SCREAMING_SNAKE_CASE       :	Dict						    =							added_tokens_extra_ids + [
                          """an_additional_special_token"""
                      ]

                      with open(os.path.join(A__    ,						"special_tokens_map.json"							)    ,						"w"    ,						encoding="utf-8"							) as outfile:
                           json.dump(A__    ,						A__							)
                      with open(os.path.join(A__    ,						"tokenizer_config.json"							)    ,						"w"    ,						encoding="utf-8"							) as outfile:
                           json.dump(A__    ,						A__							)

                      # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                      # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                      # "special_tokens_map.json" files
                      _SCREAMING_SNAKE_CASE       :	List[str]						    =							tokenizer_class.from_pretrained(
                          A__    ,						)
                      self.assertIn(
                          "an_additional_special_token"    ,						tokenizer_without_change_in_init.additional_special_tokens							)
                      self.assertEqual(
                          ["an_additional_special_token"]    ,						tokenizer_without_change_in_init.convert_ids_to_tokens(
                              tokenizer_without_change_in_init.convert_tokens_to_ids(["an_additional_special_token"]							)							)    ,						)

                      # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                      _SCREAMING_SNAKE_CASE       :	Tuple						    =							added_tokens_extra_ids + [AddedToken("a_new_additional_special_token"    ,						lstrip=A__							)]
                      _SCREAMING_SNAKE_CASE       :	Tuple						    =							tokenizer_class.from_pretrained(
                          A__    ,						additional_special_tokens=A__    ,						)

                      self.assertIn("a_new_additional_special_token"    ,						tokenizer.additional_special_tokens							)
                      self.assertEqual(
                          ["a_new_additional_special_token"]    ,						tokenizer.convert_ids_to_tokens(
                              tokenizer.convert_tokens_to_ids(["a_new_additional_special_token"]							)							)    ,						)




       def     UpperCamelCase_			(      self							)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	Any						    =							self.perceiver_tokenizer
            self.assertEqual(tokenizer.decode([1_7_8]							)    ,						"�"							)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            pass




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            pass




       def     UpperCamelCase_			(      self							)			->     int:
            pass




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass







       def     UpperCamelCase_			(      self							)			->     int:
            # The default common tokenizer tests uses invalid tokens for Perceiver that can only accept one-character
            # strings and special added tokens as tokens
            _SCREAMING_SNAKE_CASE       :	int						    =							self.get_tokenizers(fast=A__    ,						do_lower_case=A__							)
            for tokenizer in tokenizers:
                 with self.subTest(F"""{tokenizer.__class__.__name__}"""							):
                      _SCREAMING_SNAKE_CASE       :	Tuple						    =							["""[CLS]""", """t""", """h""", """i""", """s""", """ """, """i""", """s""", """ """, """a""", """ """, """t""", """e""", """s""", """t""", """[SEP]"""]
                      _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.convert_tokens_to_string(A__							)

                      self.assertIsInstance(A__    ,						A__							)






import os
import pickle
import unittest

from transformers import AutoTokenizer
from transformers.models.bert.tokenization_bert import BertTokenizer
from transformers.models.bert_japanese.tokenization_bert_japanese import (
    VOCAB_FILES_NAMES,
    BertJapaneseTokenizer,
    CharacterTokenizer,
    JumanppTokenizer,
    MecabTokenizer,
    SudachiTokenizer,
    WordpieceTokenizer,
)
from transformers.testing_utils import custom_tokenizers, require_jumanpp, require_sudachi

from ...test_tokenization_common import TokenizerTesterMixin





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False
       __snake_case 			=    True




       def     UpperCamelCase_			(      self							)			->     Dict:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	Dict						    =							[
                "[UNK]",
                "[CLS]",
                "[SEP]",
                "こんにちは",
                "こん",
                "にちは",
                "ばんは",
                "##こん",
                "##にちは",
                "##ばんは",
                "世界",
                "##世界",
                "、",
                "##、",
                "。",
                "##。",
            ]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	str						    =							"こんにちは 、 世界 。 こんばんは 、 世界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Dict:
            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	int						    =							self.get_input_output_texts(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode(__lowerCamelCase    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.decode(__lowerCamelCase    ,						clean_up_tokenization_spaces=__lowerCamelCase							)
            return text, ids




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     List[str]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							self.tokenizer_class(self.vocab_file							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.tokenize("こんにちは、世界。\nこんばんは、世界。"							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="mecab"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     str:
            try:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							MecabTokenizer(mecab_dic="unidic_lite"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[Any]:
            try:
                 _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(mecab_dic="unidic"							)
            except ModuleNotFoundError:
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							MecabTokenizer(do_lower_case=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップルストア", "で", "iphone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     Dict:
            try:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							MecabTokenizer(
                     do_lower_case=__lowerCamelCase    ,						normalize_text=__lowerCamelCase    ,						mecab_option="-d /usr/local/lib/mecab/dic/jumandic"							)
            except RuntimeError:
                 # if dict doesn't exist in the system, previous code raises this error.
                 return

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							MecabTokenizer(normalize_text=__lowerCamelCase    ,						mecab_dic="ipadic"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱｯﾌﾟﾙストア", "で", "iPhone", "８", "が", "発売", "さ", "れ", "た", "　", "。"]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="sudachi"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iPhone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="A"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国", "人", "参政", "権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     int:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="B"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人", "参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[Any]:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							SudachiTokenizer(sudachi_dict_type="core"    ,						sudachi_split_mode="C"							)

            self.assertListEqual(tokenizer.tokenize("外国人参政権"							)    ,						["外国人参政権"]							)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Dict						    =							SudachiTokenizer(do_lower_case=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "アップル", "ストア", "で", "iphone", "8", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", " ", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							SudachiTokenizer(normalize_text=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						[" ", "\t", "ｱｯﾌﾟﾙ", "ストア", "で", "iPhone", "８", " ", "が", " ", " ", "\n ", "発売", "さ", "れ", "た", "\u3000", "。", " ", " "]    ,						)




       @require_sudachi
       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							SudachiTokenizer(trim_whitespace=__lowerCamelCase    ,						sudachi_dict_type="core"							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れ", "た", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Dict:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							self.tokenizer_class(self.vocab_file    ,						word_tokenizer_type="jumanpp"							)
            self.assertIsNotNone(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							"こんにちは、世界。\nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.tokenize(__lowerCamelCase							)
            self.assertListEqual(__lowerCamelCase    ,						["こんにちは", "、", "世界", "。", "こん", "##ばんは", "、", "世界", "。"]							)
            self.assertListEqual(tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 1_2, 1_0, 1_4, 4, 9, 1_2, 1_0, 1_4]							)

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							os.path.join(self.tmpdirname    ,						"tokenizer.bin"							)
            with open(__lowerCamelCase    ,						"wb"							) as handle:
                 pickle.dump(__lowerCamelCase    ,						__lowerCamelCase							)

            with open(__lowerCamelCase    ,						"rb"							) as handle:
                 _SCREAMING_SNAKE_CASE       :	Any						    =							pickle.load(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer_new.tokenize(__lowerCamelCase							)

            self.assertListEqual(__lowerCamelCase    ,						__lowerCamelCase							)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Tuple:
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							JumanppTokenizer(do_lower_case=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iphone", "8", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							JumanppTokenizer(normalize_text=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["ｱ", "ｯ", "ﾌ", "ﾟ", "ﾙ", "ストア", "で", "iPhone", "８", "\u3000", "が", "\u3000", "\u3000", "\u3000", "発売", "さ", "れた", "\u3000", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							JumanppTokenizer(trim_whitespace=__lowerCamelCase							)

            self.assertListEqual(
                tokenizer.tokenize(" \tｱｯﾌﾟﾙストアでiPhone８ が  \n 発売された　。  "							)    ,						["アップル", "ストア", "で", "iPhone", "8", "が", "発売", "さ", "れた", "。"]    ,						)




       @require_jumanpp
       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							JumanppTokenizer()

            self.assertListEqual(
                tokenizer.tokenize("ありがとうございますm(_ _)ｍ見つけるのが大変です。"							)    ,						["ありがとう", "ございます", "m(_ _)m", "見つける", "の", "が", "大変です", "。"]    ,						)




       def     UpperCamelCase_			(      self							)			->     List[str]:
            _SCREAMING_SNAKE_CASE       :	int						    =							["[UNK]", "[CLS]", "[SEP]", "こんにちは", "こん", "にちは", "ばんは", "##こん", "##にちは", "##ばんは"]

            _SCREAMING_SNAKE_CASE       :	str						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							i
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							WordpieceTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こんにちは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは"							)    ,						["こん", "##ばんは"]							)

            self.assertListEqual(tokenizer.tokenize("こんばんは こんばんにちは こんにちは"							)    ,						["こん", "##ばんは", "[UNK]", "こんにちは"]							)




       def     UpperCamelCase_			(      self							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	int						    =							BertJapaneseTokenizer.from_pretrained("nlp-waseda/roberta-base-japanese-with-auto-jumanpp"							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.subword_tokenizer

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							subword_tokenizer.tokenize("国境 の 長い トンネル を 抜ける と 雪国 であった 。"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁国境", "▁の", "▁長い", "▁トンネル", "▁を", "▁抜ける", "▁と", "▁雪", "国", "▁であった", "▁。"]							)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							subword_tokenizer.tokenize("こんばんは こんばん にち は こんにちは"							)
            self.assertListEqual(__lowerCamelCase    ,						["▁こん", "ばん", "は", "▁こん", "ばん", "▁に", "ち", "▁は", "▁こんにちは"]							)







       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese"							)

            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( __lowercase  ,     unittest.TestCase    ):






       '''simple docstring'''


       __snake_case 			=    BertJapaneseTokenizer
       __snake_case 			=    False




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            super().setUp()

            _SCREAMING_SNAKE_CASE       :	List[str]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	int						    =							os.path.join(self.tmpdirname    ,						VOCAB_FILES_NAMES["vocab_file"]							)
            with open(self.vocab_file    ,						"w"    ,						encoding="utf-8"							) as vocab_writer:
                 vocab_writer.write("".join([x + "\n" for x in vocab_tokens]							)							)




       def     UpperCamelCase_			(      self    ,						**__lowerCamelCase							)			->     List[Any]:
            return BertJapaneseTokenizer.from_pretrained(self.tmpdirname    ,						subword_tokenizer_type="character"    ,						**__lowerCamelCase							)




       def     UpperCamelCase_			(      self    ,						__lowerCamelCase							)			->     Optional[int]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							"こんにちは、世界。 \nこんばんは、世界。"
            _SCREAMING_SNAKE_CASE       :	Dict						    =							"こ ん に ち は 、 世 界 。 こ ん ば ん は 、 世 界 。"
            return input_text, output_text




       def     UpperCamelCase_			(      self							)			->     Tuple:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     int:
            pass  # TODO add if relevant




       def     UpperCamelCase_			(      self							)			->     Any:
            _SCREAMING_SNAKE_CASE       :	int						    =							self.tokenizer_class(self.vocab_file    ,						subword_tokenizer_type="character"							)

            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.tokenize("こんにちは、世界。 \nこんばんは、世界。"							)
            self.assertListEqual(
                __lowerCamelCase    ,						["こ", "ん", "に", "ち", "は", "、", "世", "界", "。", "こ", "ん", "ば", "ん", "は", "、", "世", "界", "。"]							)
            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(__lowerCamelCase							)    ,						[3, 4, 5, 6, 7, 1_1, 9, 1_0, 1_2, 3, 4, 8, 4, 7, 1_1, 9, 1_0, 1_2]							)




       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							["[UNK]", "[CLS]", "[SEP]", "こ", "ん", "に", "ち", "は", "ば", "世", "界", "、", "。"]

            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							{}
            for i, token in enumerate(__lowerCamelCase							):
                 _SCREAMING_SNAKE_CASE       :	int						    =							i
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							CharacterTokenizer(vocab=__lowerCamelCase    ,						unk_token="[UNK]"							)

            self.assertListEqual(tokenizer.tokenize(""							)    ,						[]							)

            self.assertListEqual(tokenizer.tokenize("こんにちは"							)    ,						["こ", "ん", "に", "ち", "は"]							)

            self.assertListEqual(tokenizer.tokenize("こんにちほ"							)    ,						["こ", "ん", "に", "ち", "[UNK]"]							)







       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.tokenizer_class.from_pretrained("cl-tohoku/bert-base-japanese-char"							)

            _SCREAMING_SNAKE_CASE       :	int						    =							tokenizer.encode("ありがとう。"    ,						add_special_tokens=__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							tokenizer.encode("どういたしまして。"    ,						add_special_tokens=__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	str						    =							tokenizer.build_inputs_with_special_tokens(__lowerCamelCase    ,						__lowerCamelCase							)

            # 2 is for "[CLS]", 3 is for "[SEP]"
            assert encoded_sentence == [2] + text + [3]
            assert encoded_pair == [2] + text + [3] + text_a + [3]





@custom_tokenizers
class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     List[Any]:
            _SCREAMING_SNAKE_CASE       :	str						    =							"cl-tohoku/bert-base-japanese"
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							AutoTokenizer.from_pretrained(__lowerCamelCase							)
            self.assertIsInstance(__lowerCamelCase    ,						__lowerCamelCase							)





class 			lowerCAmelCase__( unittest.TestCase    ):






       '''simple docstring'''


       def     UpperCamelCase_			(      self							)			->     str:
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"cl-tohoku/bert-base-japanese"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							"bert-base-cased"
            with self.assertLogs("transformers"    ,						level="WARNING"							) as cm:
                 BertJapaneseTokenizer.from_pretrained(__lowerCamelCase							)
                 self.assertTrue(
                     cm.records[0].message.startswith(
                         "The tokenizer class you load from this checkpoint is not the same type as the class this function"
                         " is called from."							)							)