'''simple docstring'''







import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
		import torch


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation


UpperCamelCase_      	=	logging.get_logger(__name__)
UpperCamelCase_      	=	{'''vocab_file''': '''spiece.model'''}

UpperCamelCase_      	=	{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

UpperCamelCase_      	=	{
    '''AI-Sweden/gpt-sw3-126m''': 20_48,
    '''AI-Sweden/gpt-sw3-350m''': 20_48,
    '''AI-Sweden/gpt-sw3-1.6b''': 20_48,
    '''AI-Sweden/gpt-sw3-6.7b''': 20_48,
    '''AI-Sweden/gpt-sw3-20b''': 20_48,
}




class _SCREAMING_SNAKE_CASE(							__a					):
						A_			:							Union[str, Any]     		= VOCAB_FILES_NAMES
						A_			:							Any     		= PRETRAINED_VOCAB_FILES_MAP
						A_			:							List[str]     		= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						A_			:							List[str]     		= ['input_ids', 'attention_mask']




						def __init__(    self :       str							,							UpperCamelCase_ :       List[str]							,							UpperCamelCase_ :       List[str]=False							,							UpperCamelCase_ :       List[str]=False							,							UpperCamelCase_ :       Optional[int]=False							,							UpperCamelCase_ :       Dict=None							,							UpperCamelCase_ :       Union[str, Any]=None							,							UpperCamelCase_ :       List[Any]=None							,							UpperCamelCase_ :       Union[str, Any]=None							,							UpperCamelCase_ :       Optional[Dict[str, Any]] = None							,							**UpperCamelCase_ :       Tuple							,							)     ->				Any:
													SCREAMING_SNAKE_CASE__     :Optional[Any]						    = {} if sp_model_kwargs is None else sp_model_kwargs

													SCREAMING_SNAKE_CASE__     :Dict						    = kwargs.get('name_or_path'      )
													if name_or_path is None:
																				logger.warning(
																				    'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
																				    ' you are testing the model, this can safely be ignored'      )
																				SCREAMING_SNAKE_CASE__     :int						    = 'None'

													# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
													SCREAMING_SNAKE_CASE__     :Dict						    = '<|endoftext|>' if eos_token is None else eos_token
													SCREAMING_SNAKE_CASE__     :List[Any]						    = '<unk>' if unk_token is None else unk_token
													if "gpt-sw3-7b" in name_or_path:
																				SCREAMING_SNAKE_CASE__     :str						    = unk_token if pad_token is None else pad_token
																				SCREAMING_SNAKE_CASE__     :str						    = eos_token if bos_token is None else bos_token
													else:
																				SCREAMING_SNAKE_CASE__     :Dict						    = '<pad>' if pad_token is None else pad_token
																				SCREAMING_SNAKE_CASE__     :Optional[Any]						    = '<s>' if bos_token is None else bos_token

													super().__init__(
													    do_lower_case=lowercase_							,							remove_space=lowercase_							,							keep_accents=lowercase_							,							bos_token=lowercase_							,							eos_token=lowercase_							,							unk_token=lowercase_							,							pad_token=lowercase_							,							sp_model_kwargs=self.sp_model_kwargs							,							**lowercase_							,							)

													SCREAMING_SNAKE_CASE__     :Optional[Any]						    = do_lower_case
													SCREAMING_SNAKE_CASE__     :Any						    = remove_space
													SCREAMING_SNAKE_CASE__     :Any						    = keep_accents
													SCREAMING_SNAKE_CASE__     :Optional[Any]						    = vocab_file

													SCREAMING_SNAKE_CASE__     :Optional[Any]						    = spm.SentencePieceProcessor(**self.sp_model_kwargs      )
													self.sp_model.Load(lowercase_      )

													# Used for whitespace normalization in input texts
													# fmt : off
													SCREAMING_SNAKE_CASE__     :List[Any]						    = {' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
													# fmt : on

													# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
													SCREAMING_SNAKE_CASE__     :int						    = re.compile(
													    f'''[{"".join(map(lowercase_							,							list(range(0							,							9      )      ) + list(range(11							,							32      )      ) + list(range(1_27							,							1_60      )      ) + [1_60, 1_73, 82_03]      )      )}]'''      )




						def __getstate__(    self :       Tuple      )     ->				Any:
													SCREAMING_SNAKE_CASE__     :Dict						    = self.__dict__.copy()
													SCREAMING_SNAKE_CASE__     :Dict						    = None
													return state




						def __setstate__(    self :       Tuple							,							UpperCamelCase_ :       Union[str, Any]      )     ->				Any:
													SCREAMING_SNAKE_CASE__     :int						    = d

													# for backward compatibility
													if not hasattr(self							,							'sp_model_kwargs'      ):
																				SCREAMING_SNAKE_CASE__     :Dict						    = {}

													SCREAMING_SNAKE_CASE__     :Optional[int]						    = spm.SentencePieceProcessor(**self.sp_model_kwargs      )
													self.sp_model.Load(self.vocab_file      )




						@property
						# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
						def      __lowerCamelCase       (    self :       int      )     ->				int:
													return len(self.sp_model      )




						def      __lowerCamelCase       (    self :       List[Any]							,							UpperCamelCase_ :       str      )     ->				str:
													SCREAMING_SNAKE_CASE__     :Tuple						    = self.non_printing_characters_re.sub(''							,							lowercase_      )

													# Normalize whitespaces
													SCREAMING_SNAKE_CASE__     :Tuple						    = ''.join([char if char not in self.whitespaces else ' ' for char in text]      )

													# NFC Unicode normalization
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = unicodedata.normalize('NFC'							,							lowercase_      )
													return text




						def      __lowerCamelCase       (    self :       Any							,							UpperCamelCase_ :       str							,							**UpperCamelCase_ :       Any      )     ->				Optional[int]:
													SCREAMING_SNAKE_CASE__     :Tuple						    = self.preprocess_text(lowercase_      )
													return self.sp_model.encode(lowercase_							,							out_type=lowercase_      )




						def      __lowerCamelCase       (    self :       Dict							,							UpperCamelCase_ :       str      )     ->				Tuple:
													return self.sp_model.PieceToId(lowercase_      )




						def      __lowerCamelCase       (    self :       Any							,							UpperCamelCase_ :       int      )     ->				Any:
													return self.sp_model.IdToPiece(lowercase_      )




						@staticmethod
						def      __lowerCamelCase       (    UpperCamelCase_ :       str      )     ->				List[Any]:
													return out_string




						def      __lowerCamelCase       (    self :       Dict							,							UpperCamelCase_ :       List[str]      )     ->				Dict:
													SCREAMING_SNAKE_CASE__     :str						    = []
													SCREAMING_SNAKE_CASE__     :Optional[int]						    = ''
													SCREAMING_SNAKE_CASE__     :Optional[Any]						    = False
													for token in tokens:
																				# make sure that special tokens are not decoded using sentencepiece model
																				if token in self.all_special_tokens:
																											# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																											if not prev_is_special:
																																		out_string += " "

																											out_string += self.sp_model.decode(lowercase_      ) + token
																											SCREAMING_SNAKE_CASE__     :Any						    = True
																											SCREAMING_SNAKE_CASE__     :Any						    = []
																				else:
																											current_sub_tokens.append(lowercase_      )
																											SCREAMING_SNAKE_CASE__     :Optional[Any]						    = False
													out_string += self.sp_model.decode(lowercase_      )

													return out_string




						def      __lowerCamelCase       (    self :       Tuple      )     ->				Optional[Any]:
													SCREAMING_SNAKE_CASE__     :Dict						    = {self.convert_ids_to_tokens(lowercase_      ): i for i in range(self.vocab_size      )}
													vocab.update(self.added_tokens_encoder      )
													return vocab




						def      __lowerCamelCase       (    self :       Tuple							,							UpperCamelCase_ :       str							,							UpperCamelCase_ :       Optional[str] = None      )     ->				str:


													if not os.path.isdir(lowercase_      ):
																				logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''      )
																				return
													SCREAMING_SNAKE_CASE__     :str						    = os.path.join(
													    lowercase_							,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

													if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowercase_      ) and os.path.isfile(self.vocab_file      ):
																				copyfile(self.vocab_file							,							lowercase_      )
													elif not os.path.isfile(self.vocab_file      ):
																				with open(lowercase_							,							'wb'      ) as fi:
																											SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = self.sp_model.serialized_model_proto()
																											fi.write(lowercase_      )

													return (out_vocab_file,)




						def      __lowerCamelCase       (    self :       Any							,							UpperCamelCase_ :       Union[str, List[str]]							,							UpperCamelCase_ :       Union[str, bool] = False      )     ->				Optional[int]:


													if isinstance(lowercase_							,							lowercase_      ):
																				SCREAMING_SNAKE_CASE__     :int						    = self.preprocess_text(lowercase_      )
																				SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = self.sp_model.encode(lowercase_      )
													else:
																				SCREAMING_SNAKE_CASE__     :List[str]						    = [self.preprocess_text(lowercase_      ) for t in text]
																				SCREAMING_SNAKE_CASE__     :Optional[Any]						    = self.sp_model.encode(lowercase_      )

													if return_tensors is True or return_tensors == "pt":
																				SCREAMING_SNAKE_CASE__     :List[Any]						    = torch.tensor(lowercase_      )

													return token_ids




						def      __lowerCamelCase       (    self :       Union[str, Any]							,							UpperCamelCase_ :       Union[int, List[int]]      )     ->				Union[str, Any]:
													return self.sp_model.decode(lowercase_      )




						def      __lowerCamelCase       (    self :       Any							,							UpperCamelCase_ :       "Conversation"      )     ->				Optional[int]:
													SCREAMING_SNAKE_CASE__     :Tuple						    = [f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
													SCREAMING_SNAKE_CASE__     :Union[str, Any]						    = (
													    f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(lowercase_      ) + f'''{self.bos_token}Bot:'''
													)
													return self.encode(text=lowercase_      )







'''simple docstring'''


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
       import torch


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


__snake_case						  =		logging.get_logger(__name__)
__snake_case						  =		{"""vocab_file""": """spiece.model"""}

__snake_case						  =		{
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

__snake_case						  =		{
    """AI-Sweden/gpt-sw3-126m""": 2048,
    """AI-Sweden/gpt-sw3-350m""": 2048,
    """AI-Sweden/gpt-sw3-1.6b""": 2048,
    """AI-Sweden/gpt-sw3-6.7b""": 2048,
    """AI-Sweden/gpt-sw3-20b""": 2048,
}





class       _a    (		__a      ):


  """simple docstring"""

  A_    							=						VOCAB_FILES_NAMES
  A_    							=						PRETRAINED_VOCAB_FILES_MAP
  A_    							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  A_    							=						['''input_ids''', '''attention_mask''']




  def __init__(		self      :       str   ,				lowercase_      :       List[str]   ,				lowercase_      :       List[str]=False   ,				lowercase_      :       List[str]=False   ,				lowercase_      :       Optional[int]=False   ,				lowercase_      :       Dict=None   ,				lowercase_      :       Union[str, Any]=None   ,				lowercase_      :       List[Any]=None   ,				lowercase_      :       Union[str, Any]=None   ,				lowercase_      :       Optional[Dict[str, Any]] = None   ,				**lowercase_      :       Tuple   ,				):







    '''simple docstring'''


    lowercase_     =       {} if sp_model_kwargs is None else sp_model_kwargs

    lowercase_     =       kwargs.get("""name_or_path"""      )
    if name_or_path is None:
      logger.warning(
          """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
          """ you are testing the model, this can safely be ignored"""      )
      lowercase_     =       """None"""

    # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
    lowercase_     =       """<|endoftext|>""" if eos_token is None else eos_token
    lowercase_     =       """<unk>""" if unk_token is None else unk_token
    if "gpt-sw3-7b" in name_or_path:
      lowercase_     =       unk_token if pad_token is None else pad_token
      lowercase_     =       eos_token if bos_token is None else bos_token
    else:
      lowercase_     =       """<pad>""" if pad_token is None else pad_token
      lowercase_     =       """<s>""" if bos_token is None else bos_token

    super().__init__(
        do_lower_case=lowercase_   ,				remove_space=lowercase_   ,				keep_accents=lowercase_   ,				bos_token=lowercase_   ,				eos_token=lowercase_   ,				unk_token=lowercase_   ,				pad_token=lowercase_   ,				sp_model_kwargs=self.sp_model_kwargs   ,				**lowercase_   ,				)

    lowercase_     =       do_lower_case
    lowercase_     =       remove_space
    lowercase_     =       keep_accents
    lowercase_     =       vocab_file

    lowercase_     =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
    self.sp_model.Load(lowercase_      )

    # Used for whitespace normalization in input texts
    # fmt : off
    lowercase_     =       {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
    # fmt : on

    # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
    lowercase_     =       re.compile(
        F"""[{"".join(map(lowercase_   ,				list(range(0   ,				9      )      ) + list(range(11   ,				32      )      ) + list(range(127   ,				160      )      ) + [160, 173, 8_203]      )      )}]"""      )




  def __getstate__(		self      :       Tuple      ):







    '''simple docstring'''


    lowercase_     =       self.__dict__.copy()
    lowercase_     =       None
    return state




  def __setstate__(		self      :       Tuple   ,				lowercase_      :       Union[str, Any]      ):







    '''simple docstring'''


    lowercase_     =       d

    # for backward compatibility
    if not hasattr(self   ,				"""sp_model_kwargs"""      ):
      lowercase_     =       {}

    lowercase_     =       spm.SentencePieceProcessor(**self.sp_model_kwargs      )
    self.sp_model.Load(self.vocab_file      )




  @property
  # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
  def 		lowerCamelCase__  (		self      :       int      ):







    '''simple docstring'''


    return len(self.sp_model      )




  def 		lowerCamelCase__  (		self      :       List[Any]   ,				lowercase_      :       str      ):







    '''simple docstring'''


    lowercase_     =       self.non_printing_characters_re.sub(""""""   ,				lowercase_      )

    # Normalize whitespaces
    lowercase_     =       """""".join([char if char not in self.whitespaces else """ """ for char in text]      )

    # NFC Unicode normalization
    lowercase_     =       unicodedata.normalize("""NFC"""   ,				lowercase_      )
    return text




  def 		lowerCamelCase__  (		self      :       Any   ,				lowercase_      :       str   ,				**lowercase_      :       Any      ):







    '''simple docstring'''


    lowercase_     =       self.preprocess_text(lowercase_      )
    return self.sp_model.encode(lowercase_   ,				out_type=lowercase_      )




  def 		lowerCamelCase__  (		self      :       Dict   ,				lowercase_      :       str      ):







    '''simple docstring'''


    return self.sp_model.PieceToId(lowercase_      )




  def 		lowerCamelCase__  (		self      :       Any   ,				lowercase_      :       int      ):







    '''simple docstring'''


    return self.sp_model.IdToPiece(lowercase_      )




  @staticmethod
  def 		lowerCamelCase__  (		lowercase_      :       str      ):







    '''simple docstring'''


    return out_string




  def 		lowerCamelCase__  (		self      :       Dict   ,				lowercase_      :       List[str]      ):







    '''simple docstring'''


    lowercase_     =       []
    lowercase_     =       """"""
    lowercase_     =       False
    for token in tokens:
      # make sure that special tokens are not decoded using sentencepiece model
      if token in self.all_special_tokens:
        # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
        if not prev_is_special:
          out_string += " "

        out_string += self.sp_model.decode(lowercase_      ) + token
        lowercase_     =       True
        lowercase_     =       []
      else:
        current_sub_tokens.append(lowercase_      )
        lowercase_     =       False
    out_string += self.sp_model.decode(lowercase_      )

    return out_string




  def 		lowerCamelCase__  (		self      :       Tuple      ):







    '''simple docstring'''


    lowercase_     =       {self.convert_ids_to_tokens(lowercase_      ): i for i in range(self.vocab_size      )}
    vocab.update(self.added_tokens_encoder      )
    return vocab




  def 		lowerCamelCase__  (		self      :       Tuple   ,				lowercase_      :       str   ,				lowercase_      :       Optional[str] = None      ):







    '''simple docstring'''


    if not os.path.isdir(lowercase_      ):
      logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
      return
    lowercase_     =       os.path.join(
        lowercase_   ,				(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )

    if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowercase_      ) and os.path.isfile(self.vocab_file      ):
      copyfile(self.vocab_file   ,				lowercase_      )
    elif not os.path.isfile(self.vocab_file      ):
      with open(lowercase_   ,				"""wb"""      ) as fi:
        lowercase_     =       self.sp_model.serialized_model_proto()
        fi.write(lowercase_      )

    return (out_vocab_file,)




  def 		lowerCamelCase__  (		self      :       Any   ,				lowercase_      :       Union[str, List[str]]   ,				lowercase_      :       Union[str, bool] = False      ):







    '''simple docstring'''


    if isinstance(lowercase_   ,				lowercase_      ):
      lowercase_     =       self.preprocess_text(lowercase_      )
      lowercase_     =       self.sp_model.encode(lowercase_      )
    else:
      lowercase_     =       [self.preprocess_text(lowercase_      ) for t in text]
      lowercase_     =       self.sp_model.encode(lowercase_      )

    if return_tensors is True or return_tensors == "pt":
      lowercase_     =       torch.tensor(lowercase_      )

    return token_ids




  def 		lowerCamelCase__  (		self      :       Union[str, Any]   ,				lowercase_      :       Union[int, List[int]]      ):







    '''simple docstring'''


    return self.sp_model.decode(lowercase_      )





  def 		lowerCamelCase__  (		self      :       Any   ,				lowercase_      :       "Conversation"      ):







    '''simple docstring'''


    lowercase_     =       [F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
    lowercase_     =       (
        F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(lowercase_      ) + F"""{self.bos_token}Bot:"""
    )
    return self.encode(text=lowercase_      )

