


'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


lowercase__				     =		logging.get_logger(__name__)
lowercase__				     =		{"vocab_file": "spiece.model"}

lowercase__				     =		{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

lowercase__				     =		{
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}





class 						__SCREAMING_SNAKE_CASE					(      a__ ):
     UpperCAmelCase 					=				VOCAB_FILES_NAMES
     UpperCAmelCase 					=				PRETRAINED_VOCAB_FILES_MAP
     UpperCAmelCase 					=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     UpperCAmelCase 					=				['''input_ids''', '''attention_mask''']






     def __init__(			self     ,							__UpperCamelCase     ,							__UpperCamelCase=False     ,							__UpperCamelCase=False     ,							__UpperCamelCase=False     ,							__UpperCamelCase=None     ,							__UpperCamelCase=None     ,							__UpperCamelCase=None     ,							__UpperCamelCase=None     ,							__UpperCamelCase = None     ,							**__UpperCamelCase     ,							)       ->    List[str]:
          _a						=		{} if sp_model_kwargs is None else sp_model_kwargs

          _a						=		kwargs.get("name_or_path"		)
          if name_or_path is None:
               logger.warning(
                   "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
                   " you are testing the model, this can safely be ignored"		)
               _a						=		'None'

          # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
          _a						=		'<|endoftext|>' if eos_token is None else eos_token
          _a						=		'<unk>' if unk_token is None else unk_token
          if "gpt-sw3-7b" in name_or_path:
               _a						=		unk_token if pad_token is None else pad_token
               _a						=		eos_token if bos_token is None else bos_token
          else:
               _a						=		'<pad>' if pad_token is None else pad_token
               _a						=		'<s>' if bos_token is None else bos_token

          super().__init__(
              do_lower_case=_A     ,							remove_space=_A     ,							keep_accents=_A     ,							bos_token=_A     ,							eos_token=_A     ,							unk_token=_A     ,							pad_token=_A     ,							sp_model_kwargs=self.sp_model_kwargs     ,							**_A     ,							)

          _a						=		do_lower_case
          _a						=		remove_space
          _a						=		keep_accents
          _a						=		vocab_file

          _a						=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(_A		)

          # Used for whitespace normalization in input texts
          # fmt : off
          _a						=		{' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
          # fmt : on

          # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
          _a						=		re.compile(
              f"[{''.join(map(_A     ,							list(range(0     ,							9		)		) + list(range(11     ,							32		)		) + list(range(127     ,							160		)		) + [160, 173, 8_203]		)		)}]"		)






     def __getstate__(			self		)       ->    Tuple:
          _a						=		self.__dict__.copy()
          _a						=		None
          return state






     def __setstate__(			self     ,							__UpperCamelCase		)       ->    Dict:
          _a						=		d

          # for backward compatibility
          if not hasattr(self     ,							"sp_model_kwargs"		):
               _a						=		{}

          _a						=		spm.SentencePieceProcessor(**self.sp_model_kwargs		)
          self.sp_model.Load(self.vocab_file		)






     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def  a_					(			self		)       ->    Union[str, Any]:
          return len(self.sp_model		)






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Dict:
          _a						=		self.non_printing_characters_re.sub(""     ,							_A		)

          # Normalize whitespaces
          _a						=		''.join([char if char not in self.whitespaces else " " for char in text]		)

          # NFC Unicode normalization
          _a						=		unicodedata.normalize("NFC"     ,							_A		)
          return text






     def  a_					(			self     ,							__UpperCamelCase     ,							**__UpperCamelCase		)       ->    List[Any]:
          _a						=		self.preprocess_text(_A		)
          return self.sp_model.encode(_A     ,							out_type=_A		)






     def  a_					(			self     ,							__UpperCamelCase		)       ->    str:
          return self.sp_model.PieceToId(_A		)






     def  a_					(			self     ,							__UpperCamelCase		)       ->    int:
          return self.sp_model.IdToPiece(_A		)






     @staticmethod
     def  a_					(			__UpperCamelCase		)       ->    Dict:
          return out_string






     def  a_					(			self     ,							__UpperCamelCase		)       ->    int:
          _a						=		[]
          _a						=		''
          _a						=		False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                    if not prev_is_special:
                         out_string += " "

                    out_string += self.sp_model.decode(_A		) + token
                    _a						=		True
                    _a						=		[]
               else:
                    current_sub_tokens.append(_A		)
                    _a						=		False
          out_string += self.sp_model.decode(_A		)

          return out_string






     def  a_					(			self		)       ->    Tuple:
          _a						=		{self.convert_ids_to_tokens(_A		): i for i in range(self.vocab_size		)}
          vocab.update(self.added_tokens_encoder		)
          return vocab






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = None		)       ->    Optional[int]:

          if not os.path.isdir(_A		):
               logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
               return
          _a						=		os.path.join(
              _A     ,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]		)

          if os.path.abspath(self.vocab_file		) != os.path.abspath(_A		) and os.path.isfile(self.vocab_file		):
               copyfile(self.vocab_file     ,							_A		)
          elif not os.path.isfile(self.vocab_file		):
               with open(_A     ,							"wb"		) as fi:
                    _a						=		self.sp_model.serialized_model_proto()
                    fi.write(_A		)

          return (out_vocab_file,)






     def  a_					(			self     ,							__UpperCamelCase     ,							__UpperCamelCase = False		)       ->    Dict:

          if isinstance(_A     ,							_A		):
               _a						=		self.preprocess_text(_A		)
               _a						=		self.sp_model.encode(_A		)
          else:
               _a						=		[self.preprocess_text(_A		) for t in text]
               _a						=		self.sp_model.encode(_A		)

          if return_tensors is True or return_tensors == "pt":
               _a						=		torch.tensor(_A		)

          return token_ids






     def  a_					(			self     ,							__UpperCamelCase		)       ->    Tuple:
          return self.sp_model.decode(_A		)






     def  a_					(			self     ,							__UpperCamelCase		)       ->    int:
          _a						=		[f"User: {text}" if is_user else f"Bot: {text}" for is_user, text in conversation.iter_texts()]
          _a						=		(
              f"{self.eos_token}{self.bos_token}" + f"{self.bos_token}".join(_A		) + f"{self.bos_token}Bot:"
          )
          return self.encode(text=_A		)









'''simple docstring'''

lowercase__				     =		65_521






def __UpperCamelCase   (			__lowerCamelCase		: str				)      ->     int:







     '''simple docstring'''






     _a						=		1
     _a						=		0
     for plain_chr in plain_text:
          _a						=		(a + ord(__lowerCamelCase				)) % MOD_ADLER
          _a						=		(b + a) % MOD_ADLER
     return (b << 16) | a

