




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


__lowerCAmelCase			=							logging.get_logger(__name__)
__lowerCAmelCase			=							{'''vocab_file''': '''spiece.model'''}

__lowerCAmelCase			=							{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

__lowerCAmelCase			=							{
    '''AI-Sweden/gpt-sw3-126m''': 20_48,
    '''AI-Sweden/gpt-sw3-350m''': 20_48,
    '''AI-Sweden/gpt-sw3-1.6b''': 20_48,
    '''AI-Sweden/gpt-sw3-6.7b''': 20_48,
    '''AI-Sweden/gpt-sw3-20b''': 20_48,
}





class   __a      (   __SCREAMING_SNAKE_CASE				):
 __lowercase			:							Optional[int]       					=			VOCAB_FILES_NAMES
 __lowercase			:							List[Any]       					=			PRETRAINED_VOCAB_FILES_MAP
 __lowercase			:							Optional[int]       					=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __lowercase			:							List[str]       					=			['input_ids', 'attention_mask']
 def __init__(       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__=False    ,			lowerCAmelCase__=False    ,			lowerCAmelCase__=False    ,			lowerCAmelCase__=None    ,			lowerCAmelCase__=None    ,			lowerCAmelCase__=None    ,			lowerCAmelCase__=None    ,			lowerCAmelCase__ = None    ,			**lowerCAmelCase__    ,			) ->    None:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       {} if sp_model_kwargs is None else sp_model_kwargs

        lowercase__:       Tuple        =       kwargs.get('name_or_path'			)
        if name_or_path is None:
               logger.warning(
                   'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
                   ' you are testing the model, this can safely be ignored'			)
               lowercase__:       int        =       '''None'''

        # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
        lowercase__:       Any        =       '''<|endoftext|>''' if eos_token is None else eos_token
        lowercase__:       Union[str, Any]        =       '''<unk>''' if unk_token is None else unk_token
        if "gpt-sw3-7b" in name_or_path:
               lowercase__:       Optional[Any]        =       unk_token if pad_token is None else pad_token
               lowercase__:       Tuple        =       eos_token if bos_token is None else bos_token
        else:
               lowercase__:       str        =       '''<pad>''' if pad_token is None else pad_token
               lowercase__:       Union[str, Any]        =       '''<s>''' if bos_token is None else bos_token

        super().__init__(
            do_lower_case=UpperCamelCase__    ,			remove_space=UpperCamelCase__    ,			keep_accents=UpperCamelCase__    ,			bos_token=UpperCamelCase__    ,			eos_token=UpperCamelCase__    ,			unk_token=UpperCamelCase__    ,			pad_token=UpperCamelCase__    ,			sp_model_kwargs=self.sp_model_kwargs    ,			**UpperCamelCase__    ,			)

        lowercase__:       str        =       do_lower_case
        lowercase__:       List[str]        =       remove_space
        lowercase__:       Any        =       keep_accents
        lowercase__:       List[str]        =       vocab_file

        lowercase__:       int        =       spm.SentencePieceProcessor(**self.sp_model_kwargs			)
        self.sp_model.Load(UpperCamelCase__			)

        # Used for whitespace normalization in input texts
        # fmt : off
        lowercase__:       Optional[int]        =       {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
        # fmt : on

        # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
        lowercase__:       List[Any]        =       re.compile(
            F'[{"".join(map(UpperCamelCase__    ,			list(range(0    ,			9			)			) + list(range(11    ,			32			)			) + list(range(127    ,			160			)			) + [160, 173, 8_203]			)			)}]'			)
 def __getstate__(       self			) ->    str:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       self.__dict__.copy()
        lowercase__:       Any        =       None
        return state
 def __setstate__(       self    ,			lowerCAmelCase__			) ->    Tuple:




        '''simple docstring'''
        lowercase__:       List[str]        =       d

        # for backward compatibility
        if not hasattr(self    ,			'sp_model_kwargs'			):
               lowercase__:       str        =       {}

        lowercase__:       Optional[int]        =       spm.SentencePieceProcessor(**self.sp_model_kwargs			)
        self.sp_model.Load(self.vocab_file			)
 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    int:




        '''simple docstring'''
        return len(self.sp_model			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    str:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       self.non_printing_characters_re.sub(''    ,			UpperCamelCase__			)

        # Normalize whitespaces
        lowercase__:       Union[str, Any]        =       ''''''.join([char if char not in self.whitespaces else ' ' for char in text]			)

        # NFC Unicode normalization
        lowercase__:       List[Any]        =       unicodedata.normalize('NFC'    ,			UpperCamelCase__			)
        return text
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			**lowerCAmelCase__			) ->    List[str]:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       self.preprocess_text(UpperCamelCase__			)
        return self.sp_model.encode(UpperCamelCase__    ,			out_type=UpperCamelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    int:




        '''simple docstring'''
        return self.sp_model.PieceToId(UpperCamelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    str:




        '''simple docstring'''
        return self.sp_model.IdToPiece(UpperCamelCase__			)
 @staticmethod
 def 		SCREAMING_SNAKE_CASE__       (       lowerCAmelCase__			) ->    str:




        '''simple docstring'''
        return out_string
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    str:




        '''simple docstring'''
        lowercase__:       Optional[int]        =       []
        lowercase__:       str        =       ''''''
        lowercase__:       Optional[Any]        =       False
        for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                      # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                      if not prev_is_special:
                             out_string += " "

                      out_string += self.sp_model.decode(UpperCamelCase__			) + token
                      lowercase__:       Union[str, Any]        =       True
                      lowercase__:       str        =       []
               else:
                      current_sub_tokens.append(UpperCamelCase__			)
                      lowercase__:       Tuple        =       False
        out_string += self.sp_model.decode(UpperCamelCase__			)

        return out_string
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    Dict[str, int]:




        '''simple docstring'''
        lowercase__:       Optional[int]        =       {self.convert_ids_to_tokens(UpperCamelCase__			): i for i in range(self.vocab_size			)}
        vocab.update(self.added_tokens_encoder			)
        return vocab
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None			) ->    Tuple[str]:




        '''simple docstring'''







        if not os.path.isdir(UpperCamelCase__			):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory'			)
               return
        lowercase__:       str        =       os.path.join(
            UpperCamelCase__    ,			(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

        if os.path.abspath(self.vocab_file			) != os.path.abspath(UpperCamelCase__			) and os.path.isfile(self.vocab_file			):
               copyfile(self.vocab_file    ,			UpperCamelCase__			)
        elif not os.path.isfile(self.vocab_file			):
               with open(UpperCamelCase__    ,			'wb'			) as fi:
                      lowercase__:       Optional[int]        =       self.sp_model.serialized_model_proto()
                      fi.write(UpperCamelCase__			)

        return (out_vocab_file,)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = False			) ->    Union[List[int], List[List[int]], "torch.Tensor"]:




        '''simple docstring'''







        if isinstance(UpperCamelCase__    ,			UpperCamelCase__			):
               lowercase__:       Optional[int]        =       self.preprocess_text(UpperCamelCase__			)
               lowercase__:       Union[str, Any]        =       self.sp_model.encode(UpperCamelCase__			)
        else:
               lowercase__:       List[str]        =       [self.preprocess_text(UpperCamelCase__			) for t in text]
               lowercase__:       List[Any]        =       self.sp_model.encode(UpperCamelCase__			)

        if return_tensors is True or return_tensors == "pt":
               lowercase__:       Tuple        =       torch.tensor(UpperCamelCase__			)

        return token_ids
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    str:




        '''simple docstring'''
        return self.sp_model.decode(UpperCamelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__			) ->    List[int]:




        '''simple docstring'''
        lowercase__:       Union[str, Any]        =       [F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
        lowercase__:       Dict        =       (
            F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(UpperCamelCase__			) + F'{self.bos_token}Bot:'
        )
        return self.encode(text=UpperCamelCase__			)






import os
from typing import Dict, List, Union

import tensorflow as tf
from keras_nlp.tokenizers import BytePairTokenizer
from tensorflow_text import pad_model_inputs

from .tokenization_gpta import GPTaTokenizer





class   __a      (   tf.keras.layers.Layer				):
 def __init__(       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None    ,			lowerCAmelCase__ = None			) ->    int:




        '''simple docstring'''
        super().__init__()
        lowercase__:       Union[str, Any]        =       pad_token_id
        lowercase__:       List[str]        =       max_length
        lowercase__:       int        =       vocab
        lowercase__:       List[Any]        =       merges
        lowercase__:       str        =       BytePairTokenizer(lowerCAmelCase__    ,			lowerCAmelCase__    ,			sequence_length=lowerCAmelCase__			)
 @classmethod
 def 		SCREAMING_SNAKE_CASE__       (       cls    ,			lowerCAmelCase__    ,			*lowerCAmelCase__    ,			**lowerCAmelCase__			) ->    Any:




        '''simple docstring'''
        lowercase__:       Tuple        =       [' '.join(lowerCAmelCase__			) for m in tokenizer.bpe_ranks.keys()]
        lowercase__:       List[Any]        =       tokenizer.get_vocab()
        return cls(lowerCAmelCase__    ,			lowerCAmelCase__    ,			*lowerCAmelCase__    ,			**lowerCAmelCase__			)
 @classmethod
 def 		SCREAMING_SNAKE_CASE__       (       cls    ,			lowerCAmelCase__    ,			*lowerCAmelCase__    ,			**lowerCAmelCase__			) ->    List[Any]:




        '''simple docstring'''
        lowercase__:       int        =       GPTaTokenizer.from_pretrained(lowerCAmelCase__    ,			*lowerCAmelCase__    ,			**lowerCAmelCase__			)
        return cls.from_tokenizer(lowerCAmelCase__    ,			*lowerCAmelCase__    ,			**lowerCAmelCase__			)
 @classmethod
 def 		SCREAMING_SNAKE_CASE__       (       cls    ,			lowerCAmelCase__			) ->    Dict:




        '''simple docstring'''
        return cls(**lowerCAmelCase__			)
 def 		SCREAMING_SNAKE_CASE__       (       self			) ->    List[Any]:




        '''simple docstring'''
        return {
            "vocab": self.vocab,
            "merges": self.merges,
            "max_length": self.max_length,
            "pad_token_id": self.pad_token_id,
        }
 def 		SCREAMING_SNAKE_CASE__       (       self    ,			lowerCAmelCase__    ,			lowerCAmelCase__ = None			) ->    Optional[Any]:




        '''simple docstring'''
        lowercase__:       Optional[Any]        =       self.tf_tokenizer(lowerCAmelCase__			)
        lowercase__:       List[Any]        =       tf.ones_like(lowerCAmelCase__			)

        if self.pad_token_id is not None:
               # pad the tokens up to max length
               lowercase__:       int        =       max_length if max_length is not None else self.max_length

               if max_length is not None:
                      lowercase__      ,						lowercase__:       List[Any]        =       pad_model_inputs(
                          lowerCAmelCase__    ,			max_seq_length=lowerCAmelCase__    ,			pad_value=self.pad_token_id			)

        return {"attention_mask": attention_mask, "input_ids": input_ids}


