






import copy
import json
import os
import tempfile

from transformers import is_torch_available

from .test_configuration_utils import config_common_kwargs




class 			snake_case__   (							lowercase_):

       '''simple docstring'''



       def __init__(							self			,						a__			,						a__=None			,						a__=True			,						a__=None			,						**a__    ) ->      Optional[Any]:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						parent
            __snake_case			:str          =						config_class
            __snake_case			:Any          =						has_text_modality
            __snake_case			:List[Any]          =						kwargs
            __snake_case			:Optional[int]          =						common_properties



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.config_class(**self.inputs_dict    )
            __snake_case			:int          =						(
                ["""hidden_size""", """num_attention_heads""", """num_hidden_layers"""]
                if self.common_properties is None
                else self.common_properties
            )

            # Add common fields for text models
            if self.has_text_modality:
                 common_properties.extend(["""vocab_size"""]    )

            # Test that config has the common properties as getters
            for prop in common_properties:
                 self.parent.assertTrue(hasattr(a__			,						a__    )			,						msg=F'''`{prop}` does not exist'''    )

            # Test that config has the common properties as setter
            for idx, name in enumerate(a__    ):
                 try:
                      setattr(a__			,						a__			,						a__    )
                      self.parent.assertEqual(
                          getattr(a__			,						a__    )			,						a__			,						msg=F'''`{name} value {idx} expected, but was {getattr(a__			,						a__    )}'''    )
                 except NotImplementedError:
                      # Some models might not be able to implement setters for common_properties
                      # In that case, a NotImplementedError is raised
                      pass

        # Test if config class can be called with Config(prop_name=..)
            for idx, name in enumerate(a__    ):
                 try:
                      __snake_case			:List[Any]          =						self.config_class(**{name: idx}    )
                      self.parent.assertEqual(
                          getattr(a__			,						a__    )			,						a__			,						msg=F'''`{name} value {idx} expected, but was {getattr(a__			,						a__    )}'''    )
                 except NotImplementedError:
                      # Some models might not be able to implement setters for common_properties
                      # In that case, a NotImplementedError is raised
                      pass



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						self.config_class(**self.inputs_dict    )
            __snake_case			:Tuple          =						json.loads(config.to_json_string()    )
            for key, value in self.inputs_dict.items():
                 self.parent.assertEqual(obj[key]			,						a__    )



       def 					__lowercase						(							self    ) ->      Tuple:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						self.config_class(**self.inputs_dict    )

            with tempfile.TemporaryDirectory() as tmpdirname:
                 __snake_case			:int          =						os.path.join(a__			,						"""config.json"""    )
                 config_first.to_json_file(a__    )
                 __snake_case			:Optional[int]          =						self.config_class.from_json_file(a__    )

            self.parent.assertEqual(config_second.to_dict()			,						config_first.to_dict()    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.config_class(**self.inputs_dict    )

            with tempfile.TemporaryDirectory() as tmpdirname:
                 config_first.save_pretrained(a__    )
                 __snake_case			:Dict          =						self.config_class.from_pretrained(a__    )

            self.parent.assertEqual(config_second.to_dict()			,						config_first.to_dict()    )



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.config_class(**self.inputs_dict    )

            __snake_case			:Union[str, Any]          =						"""test"""
            with tempfile.TemporaryDirectory() as tmpdirname:
                 __snake_case			:Optional[int]          =						os.path.join(a__			,						a__    )
                 config_first.save_pretrained(a__    )
                 __snake_case			:int          =						self.config_class.from_pretrained(a__			,						subfolder=a__    )

            self.parent.assertEqual(config_second.to_dict()			,						config_first.to_dict()    )



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            __snake_case			:Union[str, Any]          =						self.config_class(**self.inputs_dict			,						num_labels=5    )
            self.parent.assertEqual(len(config.idalabel    )			,						5    )
            self.parent.assertEqual(len(config.labelaid    )			,						5    )

            __snake_case			:int          =						3
            self.parent.assertEqual(len(config.idalabel    )			,						3    )
            self.parent.assertEqual(len(config.labelaid    )			,						3    )



       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            if self.config_class.is_composition:
                 return
            __snake_case			:List[Any]          =						self.config_class()
            self.parent.assertIsNotNone(a__    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:Dict          =						copy.deepcopy(a__    )
            __snake_case			:Optional[int]          =						self.config_class(**a__    )
            __snake_case			:str          =						[]
            for key, value in config_common_kwargs.items():
                 if key == "torch_dtype":
                      if not is_torch_available():
                           continue
                      else:
                           import torch

                           if config.torch_dtype != torch.floataa:
                                wrong_values.append(("""torch_dtype""", config.torch_dtype, torch.floataa)    )
                 elif getattr(a__			,						a__    ) != value:
                      wrong_values.append((key, getattr(a__			,						a__    ), value)    )

            if len(a__    ) > 0:
                 __snake_case			:Any          =						"""\n""".join([F'''- {v[0]}: got {v[1]} instead of {v[2]}''' for v in wrong_values]    )
                 raise ValueError(F'''The following keys were not properly set in the config:\n{errors}'''    )



       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            self.create_and_test_config_common_properties()
            self.create_and_test_config_to_json_string()
            self.create_and_test_config_to_json_file()
            self.create_and_test_config_from_and_save_pretrained()
            self.create_and_test_config_from_and_save_pretrained_subfolder()
            self.create_and_test_config_with_num_labels()
            self.check_config_can_be_init_without_params()
            self.check_config_arguments_init()










import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 			snake_case__   (							lowercase_	,    unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase							:				Optional[int] 				= GPTSanJapaneseTokenizer
       lowerCamelCase							:				Any 				= False
       lowerCamelCase							:				Optional[Any] 				= {"do_clean_text": False, "add_prefix_space": False}



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            super().setUp()

            # fmt: off
            __snake_case			:str          =						["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
            # fmt: on
            __snake_case			:Optional[Any]          =						{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
            __snake_case			:Optional[int]          =						{"""unk_token""": """<unk>"""}

            __snake_case			:Optional[Any]          =						os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES["""vocab_file"""]    )
            __snake_case			:List[str]          =						os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES["""emoji_file"""]    )
            with open(self.vocab_file			,						"""w"""			,						encoding="""utf-8"""    ) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )
            with open(self.emoji_file			,						"""w"""    ) as emoji_writer:
                 emoji_writer.write(json.dumps(a__    )    )



       def 					__lowercase						(							self			,						**a__    ) ->      Optional[Any]:






            '''simple docstring'''





            kwargs.update(self.special_tokens_map    )
            return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname			,						**a__    )



       def 					__lowercase						(							self			,						a__    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Tuple          =						"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
            __snake_case			:Union[str, Any]          =						"""こんにちは、世界。 \nこんばんは、世界。😀"""
            return input_text, output_text



       def 					__lowercase						(							self			,						a__    ) ->      List[str]:






            '''simple docstring'''





            __snake_case       , __snake_case			:Dict          =						self.get_input_output_texts(a__    )
            __snake_case			:Dict          =						tokenizer.encode(a__			,						add_special_tokens=a__    )
            __snake_case			:int          =						tokenizer.decode(a__			,						clean_up_tokenization_spaces=a__    )
            return text, ids



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            pass  # TODO add if relevant



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            pass  # TODO add if relevant



       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            pass  # TODO add if relevant



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						self.get_tokenizer()

            # Testing tokenization
            __snake_case			:int          =						"""こんにちは、世界。　こんばんは、㔺界。"""
            __snake_case			:Dict          =						["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
            __snake_case			:Optional[Any]          =						tokenizer.tokenize(a__    )
            self.assertListEqual(a__			,						a__    )

            # Testing conversion to ids without special tokens
            __snake_case			:List[str]          =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
            __snake_case			:Any          =						tokenizer.convert_tokens_to_ids(a__    )
            self.assertListEqual(a__			,						a__    )

            # Testing conversion to ids with special tokens
            __snake_case			:Tuple          =						tokens + [tokenizer.unk_token]
            __snake_case			:Optional[int]          =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
            __snake_case			:List[Any]          =						tokenizer.convert_tokens_to_ids(a__    )
            self.assertListEqual(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Tuple          =						self.get_tokenizer()

            # Testing tokenization
            __snake_case			:Optional[Any]          =						"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
            __snake_case			:Tuple          =						"""こんにちは、、、、世界。こんばんは、、、、世界。"""
            __snake_case			:str          =						tokenizer.encode(a__    )
            __snake_case			:Optional[int]          =						tokenizer.decode(a__    )
            self.assertEqual(a__			,						a__    )



       @slow
       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            # Testing tokenization
            __snake_case			:str          =						"""こんにちは、世界。"""
            __snake_case			:List[str]          =						"""こんばんは、㔺界。😀"""
            __snake_case			:Dict          =						"""こんにちは、世界。こんばんは、世界。😀"""
            __snake_case			:int          =						tokenizer.encode(prefix_text + input_text    )
            __snake_case			:Optional[Any]          =						tokenizer.encode(""""""			,						prefix_text=prefix_text + input_text    )
            __snake_case			:Any          =						tokenizer.encode(a__			,						prefix_text=a__    )
            __snake_case			:Optional[Any]          =						tokenizer.decode(a__    )
            __snake_case			:Optional[Any]          =						tokenizer.decode(a__    )
            __snake_case			:Optional[Any]          =						tokenizer.decode(a__    )
            self.assertEqual(a__			,						a__    )
            self.assertEqual(a__			,						a__    )
            self.assertEqual(a__			,						a__    )



       @slow
       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            # Testing tokenization
            __snake_case			:List[Any]          =						"""こんにちは、世界。"""
            __snake_case			:Dict          =						"""こんばんは、㔺界。😀"""

            __snake_case			:Optional[int]          =						len(tokenizer.encode(a__    )    ) - 2
            __snake_case			:Union[str, Any]          =						len(tokenizer.encode(a__    )    ) - 2

            __snake_case			:Union[str, Any]          =						[1] + [0] * (len_prefix + len_text + 1)
            __snake_case			:Optional[Any]          =						[1] * (len_prefix + len_text + 1) + [0]
            __snake_case			:Union[str, Any]          =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

            __snake_case			:int          =						tokenizer(prefix_text + input_text    ).token_type_ids
            __snake_case			:List[Any]          =						tokenizer(""""""			,						prefix_text=prefix_text + input_text    ).token_type_ids
            __snake_case			:int          =						tokenizer(a__			,						prefix_text=a__    ).token_type_ids
            self.assertListEqual(a__			,						a__    )
            self.assertListEqual(a__			,						a__    )
            self.assertListEqual(a__			,						a__    )



       @slow
       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            __snake_case			:Tuple          =						tokenizer.encode("""あンいワ"""    )
            __snake_case			:List[str]          =						tokenizer.encode(""""""			,						prefix_text="""あンいワ"""    )
            __snake_case			:int          =						tokenizer.encode("""いワ"""			,						prefix_text="""あン"""    )

            self.assertEqual(tokenizer.decode(a__    )			,						tokenizer.decode(a__    )    )
            self.assertEqual(tokenizer.decode(a__    )			,						tokenizer.decode(a__    )    )
            self.assertNotEqual(a__			,						a__    )
            self.assertNotEqual(a__			,						a__    )
            self.assertEqual(x_token_a[1]			,						x_token_a[-1]    )  # SEG token
            self.assertEqual(x_token_a[1]			,						x_token_a[3]    )  # SEG token



       @slow
       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            __snake_case			:int          =						[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
            __snake_case			:Optional[int]          =						tokenizer(a__			,						padding=a__    )
            __snake_case			:int          =						tokenizer.batch_encode_plus(a__			,						padding=a__    )

            # fmt: off
            __snake_case			:Tuple          =						[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
            __snake_case			:Any          =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
            __snake_case			:int          =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
            # fmt: on
            self.assertListEqual(x_token.input_ids			,						a__    )
            self.assertListEqual(x_token.token_type_ids			,						a__    )
            self.assertListEqual(x_token.attention_mask			,						a__    )
            self.assertListEqual(x_token_a.input_ids			,						a__    )
            self.assertListEqual(x_token_a.token_type_ids			,						a__    )
            self.assertListEqual(x_token_a.attention_mask			,						a__    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            pass



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            pass


