






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
     import torch


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


lowerCamelCase__							=     logging.get_logger(__name__)
lowerCamelCase__							=     {"""vocab_file""": """spiece.model"""}

lowerCamelCase__							=     {
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

lowerCamelCase__							=     {
    """AI-Sweden/gpt-sw3-126m""": 2048,
    """AI-Sweden/gpt-sw3-350m""": 2048,
    """AI-Sweden/gpt-sw3-1.6b""": 2048,
    """AI-Sweden/gpt-sw3-6.7b""": 2048,
    """AI-Sweden/gpt-sw3-20b""": 2048,
}




class 			snake_case__   (							lowercase_):

       '''simple docstring'''
       lowerCamelCase							:				List[str] 				= VOCAB_FILES_NAMES
       lowerCamelCase							:				List[Any] 				= PRETRAINED_VOCAB_FILES_MAP
       lowerCamelCase							:				Union[str, Any] 				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       lowerCamelCase							:				Tuple 				= ["input_ids", "attention_mask"]



       def __init__(							self			,						a__			,						a__=False			,						a__=False			,						a__=False			,						a__=None			,						a__=None			,						a__=None			,						a__=None			,						a__ = None			,						**a__			,						) ->      None:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						{} if sp_model_kwargs is None else sp_model_kwargs

            __snake_case			:str          =						kwargs.get("""name_or_path"""    )
            if name_or_path is None:
                 logger.warning(
                     """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
                     """ you are testing the model, this can safely be ignored"""    )
                 __snake_case			:Optional[int]          =						"""None"""

            # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
            __snake_case			:Dict          =						"""<|endoftext|>""" if eos_token is None else eos_token
            __snake_case			:str          =						"""<unk>""" if unk_token is None else unk_token
            if "gpt-sw3-7b" in name_or_path:
                 __snake_case			:Union[str, Any]          =						unk_token if pad_token is None else pad_token
                 __snake_case			:Optional[int]          =						eos_token if bos_token is None else bos_token
            else:
                 __snake_case			:Any          =						"""<pad>""" if pad_token is None else pad_token
                 __snake_case			:Optional[int]          =						"""<s>""" if bos_token is None else bos_token

            super().__init__(
                do_lower_case=a__			,						remove_space=a__			,						keep_accents=a__			,						bos_token=a__			,						eos_token=a__			,						unk_token=a__			,						pad_token=a__			,						sp_model_kwargs=self.sp_model_kwargs			,						**a__			,						)

            __snake_case			:Union[str, Any]          =						do_lower_case
            __snake_case			:Optional[Any]          =						remove_space
            __snake_case			:Dict          =						keep_accents
            __snake_case			:Dict          =						vocab_file

            __snake_case			:Union[str, Any]          =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
            self.sp_model.Load(a__    )

            # Used for whitespace normalization in input texts
            # fmt : off
            __snake_case			:int          =						{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
            # fmt : on

            # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
            __snake_case			:Optional[int]          =						re.compile(
                F'''[{''.join(map(a__			,						list(range(0			,						9    )    ) + list(range(11			,						32    )    ) + list(range(1_27			,						1_60    )    ) + [1_60, 1_73, 82_03]    )    )}]'''    )



       def __getstate__(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:Tuple          =						self.__dict__.copy()
            __snake_case			:Optional[Any]          =						None
            return state



       def __setstate__(							self			,						a__    ) ->      Any:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						d

            # for backward compatibility
            if not hasattr(self			,						"""sp_model_kwargs"""    ):
                 __snake_case			:Tuple          =						{}

            __snake_case			:Optional[int]          =						spm.SentencePieceProcessor(**self.sp_model_kwargs    )
            self.sp_model.Load(self.vocab_file    )



       @property
       # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
       def 					__lowercase						(							self    ) ->      int:






            '''simple docstring'''





            return len(self.sp_model    )



       def 					__lowercase						(							self			,						a__    ) ->      str:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						self.non_printing_characters_re.sub(""""""			,						a__    )

            # Normalize whitespaces
            __snake_case			:str          =						"""""".join([char if char not in self.whitespaces else """ """ for char in text]    )

            # NFC Unicode normalization
            __snake_case			:Union[str, Any]          =						unicodedata.normalize("""NFC"""			,						a__    )
            return text



       def 					__lowercase						(							self			,						a__			,						**a__    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						self.preprocess_text(a__    )
            return self.sp_model.encode(a__			,						out_type=a__    )



       def 					__lowercase						(							self			,						a__    ) ->      int:






            '''simple docstring'''





            return self.sp_model.PieceToId(a__    )



       def 					__lowercase						(							self			,						a__    ) ->      str:






            '''simple docstring'''





            return self.sp_model.IdToPiece(a__    )



       @staticmethod
       def 					__lowercase						(							a__    ) ->      str:






            '''simple docstring'''





            return out_string



       def 					__lowercase						(							self			,						a__    ) ->      str:






            '''simple docstring'''





            __snake_case			:List[str]          =						[]
            __snake_case			:Dict          =						""""""
            __snake_case			:str          =						False
            for token in tokens:
                 # make sure that special tokens are not decoded using sentencepiece model
                 if token in self.all_special_tokens:
                      # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                      if not prev_is_special:
                           out_string += " "

                      out_string += self.sp_model.decode(a__    ) + token
                      __snake_case			:Any          =						True
                      __snake_case			:Tuple          =						[]
                 else:
                      current_sub_tokens.append(a__    )
                      __snake_case			:List[Any]          =						False
            out_string += self.sp_model.decode(a__    )

            return out_string



       def 					__lowercase						(							self    ) ->      Dict[str, int]:






            '''simple docstring'''





            __snake_case			:Union[str, Any]          =						{self.convert_ids_to_tokens(a__    ): i for i in range(self.vocab_size    )}
            vocab.update(self.added_tokens_encoder    )
            return vocab



       def 					__lowercase						(							self			,						a__			,						a__ = None    ) ->      Tuple[str]:






            '''simple docstring'''





            if not os.path.isdir(a__    ):
                 logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''    )
                 return
            __snake_case			:List[str]          =						os.path.join(
                a__			,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

            if os.path.abspath(self.vocab_file    ) != os.path.abspath(a__    ) and os.path.isfile(self.vocab_file    ):
                 copyfile(self.vocab_file			,						a__    )
            elif not os.path.isfile(self.vocab_file    ):
                 with open(a__			,						"""wb"""    ) as fi:
                      __snake_case			:str          =						self.sp_model.serialized_model_proto()
                      fi.write(a__    )

            return (out_vocab_file,)



       def 					__lowercase						(							self			,						a__			,						a__ = False    ) ->      Union[List[int], List[List[int]], "torch.Tensor"]:






            '''simple docstring'''





            if isinstance(a__			,						a__    ):
                 __snake_case			:str          =						self.preprocess_text(a__    )
                 __snake_case			:Dict          =						self.sp_model.encode(a__    )
            else:
                 __snake_case			:int          =						[self.preprocess_text(a__    ) for t in text]
                 __snake_case			:Any          =						self.sp_model.encode(a__    )

            if return_tensors is True or return_tensors == "pt":
                 __snake_case			:Dict          =						torch.tensor(a__    )

            return token_ids



       def 					__lowercase						(							self			,						a__    ) ->      str:






            '''simple docstring'''





            return self.sp_model.decode(a__    )



       def 					__lowercase						(							self			,						a__    ) ->      List[int]:






            '''simple docstring'''





            __snake_case			:Dict          =						[F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
            __snake_case			:int          =						(
                F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(a__    ) + F'''{self.bos_token}Bot:'''
            )
            return self.encode(text=a__    )










import json
import os
import unittest

from transformers.models.gptsan_japanese.tokenization_gptsan_japanese import (
    VOCAB_FILES_NAMES,
    GPTSanJapaneseTokenizer,
)
from transformers.testing_utils import require_tokenizers, slow

from ...test_tokenization_common import TokenizerTesterMixin




@require_tokenizers
class 			snake_case__   (							lowercase_	,    unittest.TestCase):

       '''simple docstring'''
       lowerCamelCase							:				Optional[int] 				= GPTSanJapaneseTokenizer
       lowerCamelCase							:				Any 				= False
       lowerCamelCase							:				Optional[Any] 				= {"do_clean_text": False, "add_prefix_space": False}



       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            super().setUp()

            # fmt: off
            __snake_case			:str          =						["""こん""", """こんに""", """にちは""", """ばんは""", """世界,㔺界""", """、""", """。""", """<BR>""", """<SP>""", """<TAB>""", """<URL>""", """<EMAIL>""", """<TEL>""", """<DATE>""", """<PRICE>""", """<BLOCK>""", """<KIGOU>""", """<U2000U2BFF>""", """<|emoji1|>""", """<unk>""", """<|bagoftoken|>""", """<|endoftext|>"""]
            # fmt: on
            __snake_case			:Optional[Any]          =						{"""emoji""": {"""\ud83d\ude00""": """<|emoji1|>"""}, """emoji_inv""": {"""<|emoji1|>""": """\ud83d\ude00"""}}  # 😀
            __snake_case			:Optional[int]          =						{"""unk_token""": """<unk>"""}

            __snake_case			:Optional[Any]          =						os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES["""vocab_file"""]    )
            __snake_case			:List[str]          =						os.path.join(self.tmpdirname			,						VOCAB_FILES_NAMES["""emoji_file"""]    )
            with open(self.vocab_file			,						"""w"""			,						encoding="""utf-8"""    ) as vocab_writer:
                 vocab_writer.write("""""".join([x + """\n""" for x in vocab_tokens]    )    )
            with open(self.emoji_file			,						"""w"""    ) as emoji_writer:
                 emoji_writer.write(json.dumps(a__    )    )



       def 					__lowercase						(							self			,						**a__    ) ->      Optional[Any]:






            '''simple docstring'''





            kwargs.update(self.special_tokens_map    )
            return GPTSanJapaneseTokenizer.from_pretrained(self.tmpdirname			,						**a__    )



       def 					__lowercase						(							self			,						a__    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Tuple          =						"""こんにちは、世界。 \nこんばんは、㔺界。😀"""
            __snake_case			:Union[str, Any]          =						"""こんにちは、世界。 \nこんばんは、世界。😀"""
            return input_text, output_text



       def 					__lowercase						(							self			,						a__    ) ->      List[str]:






            '''simple docstring'''





            __snake_case       , __snake_case			:Dict          =						self.get_input_output_texts(a__    )
            __snake_case			:Dict          =						tokenizer.encode(a__			,						add_special_tokens=a__    )
            __snake_case			:int          =						tokenizer.decode(a__			,						clean_up_tokenization_spaces=a__    )
            return text, ids



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            pass  # TODO add if relevant



       def 					__lowercase						(							self    ) ->      Optional[Any]:






            '''simple docstring'''





            pass  # TODO add if relevant



       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            pass  # TODO add if relevant



       def 					__lowercase						(							self    ) ->      str:






            '''simple docstring'''





            __snake_case			:Optional[int]          =						self.get_tokenizer()

            # Testing tokenization
            __snake_case			:int          =						"""こんにちは、世界。　こんばんは、㔺界。"""
            __snake_case			:Dict          =						["""こん""", """にちは""", """、""", """世界""", """。""", """<SP>""", """こん""", """ばんは""", """、""", """㔺界""", """。"""]
            __snake_case			:Optional[Any]          =						tokenizer.tokenize(a__    )
            self.assertListEqual(a__			,						a__    )

            # Testing conversion to ids without special tokens
            __snake_case			:List[str]          =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6]
            __snake_case			:Any          =						tokenizer.convert_tokens_to_ids(a__    )
            self.assertListEqual(a__			,						a__    )

            # Testing conversion to ids with special tokens
            __snake_case			:Tuple          =						tokens + [tokenizer.unk_token]
            __snake_case			:Optional[int]          =						[0, 2, 5, 4, 6, 8, 0, 3, 5, 4, 6, 19]
            __snake_case			:List[Any]          =						tokenizer.convert_tokens_to_ids(a__    )
            self.assertListEqual(a__			,						a__    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            __snake_case			:Tuple          =						self.get_tokenizer()

            # Testing tokenization
            __snake_case			:Optional[Any]          =						"""こんにちは、<|bagoftoken|>世界。こんばんは、<|bagoftoken|>㔺界。"""
            __snake_case			:Tuple          =						"""こんにちは、、、、世界。こんばんは、、、、世界。"""
            __snake_case			:str          =						tokenizer.encode(a__    )
            __snake_case			:Optional[int]          =						tokenizer.decode(a__    )
            self.assertEqual(a__			,						a__    )



       @slow
       def 					__lowercase						(							self    ) ->      List[Any]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            # Testing tokenization
            __snake_case			:str          =						"""こんにちは、世界。"""
            __snake_case			:List[str]          =						"""こんばんは、㔺界。😀"""
            __snake_case			:Dict          =						"""こんにちは、世界。こんばんは、世界。😀"""
            __snake_case			:int          =						tokenizer.encode(prefix_text + input_text    )
            __snake_case			:Optional[Any]          =						tokenizer.encode(""""""			,						prefix_text=prefix_text + input_text    )
            __snake_case			:Any          =						tokenizer.encode(a__			,						prefix_text=a__    )
            __snake_case			:Optional[Any]          =						tokenizer.decode(a__    )
            __snake_case			:Optional[Any]          =						tokenizer.decode(a__    )
            __snake_case			:Optional[Any]          =						tokenizer.decode(a__    )
            self.assertEqual(a__			,						a__    )
            self.assertEqual(a__			,						a__    )
            self.assertEqual(a__			,						a__    )



       @slow
       def 					__lowercase						(							self    ) ->      Any:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            # Testing tokenization
            __snake_case			:List[Any]          =						"""こんにちは、世界。"""
            __snake_case			:Dict          =						"""こんばんは、㔺界。😀"""

            __snake_case			:Optional[int]          =						len(tokenizer.encode(a__    )    ) - 2
            __snake_case			:Union[str, Any]          =						len(tokenizer.encode(a__    )    ) - 2

            __snake_case			:Union[str, Any]          =						[1] + [0] * (len_prefix + len_text + 1)
            __snake_case			:Optional[Any]          =						[1] * (len_prefix + len_text + 1) + [0]
            __snake_case			:Union[str, Any]          =						[1] + [1] * (len_prefix) + [0] * (len_text + 1)

            __snake_case			:int          =						tokenizer(prefix_text + input_text    ).token_type_ids
            __snake_case			:List[Any]          =						tokenizer(""""""			,						prefix_text=prefix_text + input_text    ).token_type_ids
            __snake_case			:int          =						tokenizer(a__			,						prefix_text=a__    ).token_type_ids
            self.assertListEqual(a__			,						a__    )
            self.assertListEqual(a__			,						a__    )
            self.assertListEqual(a__			,						a__    )



       @slow
       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            __snake_case			:Optional[Any]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            __snake_case			:Tuple          =						tokenizer.encode("""あンいワ"""    )
            __snake_case			:List[str]          =						tokenizer.encode(""""""			,						prefix_text="""あンいワ"""    )
            __snake_case			:int          =						tokenizer.encode("""いワ"""			,						prefix_text="""あン"""    )

            self.assertEqual(tokenizer.decode(a__    )			,						tokenizer.decode(a__    )    )
            self.assertEqual(tokenizer.decode(a__    )			,						tokenizer.decode(a__    )    )
            self.assertNotEqual(a__			,						a__    )
            self.assertNotEqual(a__			,						a__    )
            self.assertEqual(x_token_a[1]			,						x_token_a[-1]    )  # SEG token
            self.assertEqual(x_token_a[1]			,						x_token_a[3]    )  # SEG token



       @slow
       def 					__lowercase						(							self    ) ->      Union[str, Any]:






            '''simple docstring'''





            __snake_case			:List[str]          =						self.tokenizer_class.from_pretrained("""Tanrei/GPTSAN-japanese"""    )

            __snake_case			:int          =						[["""武田信玄""", """は、"""], ["""織田信長""", """の配下の、"""]]
            __snake_case			:Optional[int]          =						tokenizer(a__			,						padding=a__    )
            __snake_case			:int          =						tokenizer.batch_encode_plus(a__			,						padding=a__    )

            # fmt: off
            __snake_case			:Tuple          =						[[3_59_93, 86_40, 2_59_48, 3_59_98, 3_06_47, 3_56_75, 3_59_99, 3_59_99], [3_59_93, 1_03_82, 98_68, 3_59_98, 3_06_46, 94_59, 3_06_46, 3_56_75]]
            __snake_case			:Any          =						[[1, 1, 1, 0, 0, 0, 0, 0], [1, 1, 1, 0, 0, 0, 0, 0]]
            __snake_case			:int          =						[[1, 1, 1, 1, 1, 1, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1]]
            # fmt: on
            self.assertListEqual(x_token.input_ids			,						a__    )
            self.assertListEqual(x_token.token_type_ids			,						a__    )
            self.assertListEqual(x_token.attention_mask			,						a__    )
            self.assertListEqual(x_token_a.input_ids			,						a__    )
            self.assertListEqual(x_token_a.token_type_ids			,						a__    )
            self.assertListEqual(x_token_a.attention_mask			,						a__    )



       def 					__lowercase						(							self    ) ->      List[str]:






            '''simple docstring'''





            pass



       def 					__lowercase						(							self    ) ->      Optional[int]:






            '''simple docstring'''





            pass


