


'''simple docstring'''






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
  import torch


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


__A   :    Union[str, Any]											= logging.get_logger(__name__)
__A   :    Dict											= {"vocab_file": "spiece.model"}

__A   :    str											= {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

__A   :    Dict											= {
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}






class   __snake_case			(  _SCREAMING_SNAKE_CASE):







    """simple docstring"""



    lowercase  					=   VOCAB_FILES_NAMES
    lowercase  					=   PRETRAINED_VOCAB_FILES_MAP
    lowercase  					=   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowercase  					=   ['input_ids', 'attention_mask']

    def __init__(  self       :    Tuple       ,		lowerCamelCase       :    str       ,		lowerCamelCase       :    List[Any]=False       ,		lowerCamelCase       :    str=False       ,		lowerCamelCase       :    Union[str, Any]=False       ,		lowerCamelCase       :    int=None       ,		lowerCamelCase       :    Tuple=None       ,		lowerCamelCase       :    Dict=None       ,		lowerCamelCase       :    Any=None       ,		lowerCamelCase       :    Optional[Dict[str, Any]] = None       ,		**lowerCamelCase       :    Dict       ,		)  ->    None:
      lowerCAmelCase_     : List[Any]     =  {} if sp_model_kwargs is None else sp_model_kwargs

      lowerCAmelCase_     : Optional[Any]     =  kwargs.get("""name_or_path"""						)
      if name_or_path is None:
        logger.warning(
            """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
            """ you are testing the model, this can safely be ignored"""						)
        lowerCAmelCase_     : Optional[Any]     =  """None"""

      # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
      lowerCAmelCase_     : Dict     =  """<|endoftext|>""" if eos_token is None else eos_token
      lowerCAmelCase_     : str     =  """<unk>""" if unk_token is None else unk_token
      if "gpt-sw3-7b" in name_or_path:
        lowerCAmelCase_     : List[str]     =  unk_token if pad_token is None else pad_token
        lowerCAmelCase_     : Optional[Any]     =  eos_token if bos_token is None else bos_token
      else:
        lowerCAmelCase_     : Optional[Any]     =  """<pad>""" if pad_token is None else pad_token
        lowerCAmelCase_     : int     =  """<s>""" if bos_token is None else bos_token

      super().__init__(
          do_lower_case=lowerCamelCase       ,		remove_space=lowerCamelCase       ,		keep_accents=lowerCamelCase       ,		bos_token=lowerCamelCase       ,		eos_token=lowerCamelCase       ,		unk_token=lowerCamelCase       ,		pad_token=lowerCamelCase       ,		sp_model_kwargs=self.sp_model_kwargs       ,		**lowerCamelCase       ,		)

      lowerCAmelCase_     : Any     =  do_lower_case
      lowerCAmelCase_     : str     =  remove_space
      lowerCAmelCase_     : Union[str, Any]     =  keep_accents
      lowerCAmelCase_     : int     =  vocab_file

      lowerCAmelCase_     : str     =  spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(lowerCamelCase						)

      # Used for whitespace normalization in input texts
      # fmt : off
      lowerCAmelCase_     : List[Any]     =  {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
      # fmt : on

      # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
      lowerCAmelCase_     : Dict     =  re.compile(
          F'[{"".join(map(lowerCamelCase       ,		list(range(0       ,		9						)						) + list(range(11       ,		32						)						) + list(range(1_27       ,		1_60						)						) + [1_60, 1_73, 82_03]						)						)}]'						)

    def __getstate__(  self       :    Any						)  ->    List[str]:
      lowerCAmelCase_     : Tuple     =  self.__dict__.copy()
      lowerCAmelCase_     : int     =  None
      return state

    def __setstate__(  self       :    List[str]       ,		lowerCamelCase       :    str						)  ->    Dict:
      lowerCAmelCase_     : Optional[int]     =  d

      # for backward compatibility
      if not hasattr(self       ,		"""sp_model_kwargs"""						):
        lowerCAmelCase_     : int     =  {}

      lowerCAmelCase_     : str     =  spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(self.vocab_file						)

    @property
    # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
    def __lowercase  (  self       :    List[str]						)  ->    int:
      return len(self.sp_model						)

    def __lowercase  (  self       :    Optional[Any]       ,		lowerCamelCase       :    str						)  ->    str:
      lowerCAmelCase_     : Optional[int]     =  self.non_printing_characters_re.sub(""""""       ,		lowerCamelCase						)

      # Normalize whitespaces
      lowerCAmelCase_     : Optional[int]     =  """""".join([char if char not in self.whitespaces else """ """ for char in text]						)

      # NFC Unicode normalization
      lowerCAmelCase_     : Optional[int]     =  unicodedata.normalize("""NFC"""       ,		lowerCamelCase						)
      return text

    def __lowercase  (  self       :    List[Any]       ,		lowerCamelCase       :    str       ,		**lowerCamelCase       :    Any						)  ->    List[str]:
      lowerCAmelCase_     : Tuple     =  self.preprocess_text(lowerCamelCase						)
      return self.sp_model.encode(lowerCamelCase       ,		out_type=lowerCamelCase						)

    def __lowercase  (  self       :    List[Any]       ,		lowerCamelCase       :    str						)  ->    int:
      return self.sp_model.PieceToId(lowerCamelCase						)

    def __lowercase  (  self       :    Union[str, Any]       ,		lowerCamelCase       :    int						)  ->    str:
      return self.sp_model.IdToPiece(lowerCamelCase						)

    @staticmethod
    def __lowercase  (  lowerCamelCase       :    str						)  ->    str:
      return out_string

    def __lowercase  (  self       :    Optional[int]       ,		lowerCamelCase       :    List[str]						)  ->    str:
      lowerCAmelCase_     : str     =  []
      lowerCAmelCase_     : int     =  """"""
      lowerCAmelCase_     : Optional[int]     =  False
      for token in tokens:
        # make sure that special tokens are not decoded using sentencepiece model
        if token in self.all_special_tokens:
          # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
          if not prev_is_special:
            out_string += " "

          out_string += self.sp_model.decode(lowerCamelCase						) + token
          lowerCAmelCase_     : Any     =  True
          lowerCAmelCase_     : Dict     =  []
        else:
          current_sub_tokens.append(lowerCamelCase						)
          lowerCAmelCase_     : str     =  False
      out_string += self.sp_model.decode(lowerCamelCase						)

      return out_string

    def __lowercase  (  self       :    int						)  ->    Dict[str, int]:
      lowerCAmelCase_     : int     =  {self.convert_ids_to_tokens(lowerCamelCase						): i for i in range(self.vocab_size						)}
      vocab.update(self.added_tokens_encoder						)
      return vocab

    def __lowercase  (  self       :    str       ,		lowerCamelCase       :    str       ,		lowerCamelCase       :    Optional[str] = None						)  ->    Tuple[str]:
      if not os.path.isdir(lowerCamelCase						):
        logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
        return
      lowerCAmelCase_     : Optional[int]     =  os.path.join(
          lowerCamelCase       ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

      if os.path.abspath(self.vocab_file						) != os.path.abspath(lowerCamelCase						) and os.path.isfile(self.vocab_file						):
        copyfile(self.vocab_file       ,		lowerCamelCase						)
      elif not os.path.isfile(self.vocab_file						):
        with open(lowerCamelCase       ,		"""wb"""						) as fi:
          lowerCAmelCase_     : List[str]     =  self.sp_model.serialized_model_proto()
          fi.write(lowerCamelCase						)

      return (out_vocab_file,)

    def __lowercase  (  self       :    Optional[Any]       ,		lowerCamelCase       :    Union[str, List[str]]       ,		lowerCamelCase       :    Union[str, bool] = False						)  ->    Union[List[int], List[List[int]], "torch.Tensor"]:

      if isinstance(lowerCamelCase       ,		lowerCamelCase						):
        lowerCAmelCase_     : Optional[Any]     =  self.preprocess_text(lowerCamelCase						)
        lowerCAmelCase_     : Tuple     =  self.sp_model.encode(lowerCamelCase						)
      else:
        lowerCAmelCase_     : Tuple     =  [self.preprocess_text(lowerCamelCase						) for t in text]
        lowerCAmelCase_     : Tuple     =  self.sp_model.encode(lowerCamelCase						)

      if return_tensors is True or return_tensors == "pt":
        lowerCAmelCase_     : Optional[int]     =  torch.tensor(lowerCamelCase						)

      return token_ids

    def __lowercase  (  self       :    List[str]       ,		lowerCamelCase       :    Union[int, List[int]]						)  ->    str:
      return self.sp_model.decode(lowerCamelCase						)





    def __lowercase  (  self       :    List[Any]       ,		lowerCamelCase       :    "Conversation"						)  ->    List[int]:
      lowerCAmelCase_     : Optional[int]     =  [F'User: {text}' if is_user else F'Bot: {text}' for is_user, text in conversation.iter_texts()]
      lowerCAmelCase_     : Tuple     =  (
          F'{self.eos_token}{self.bos_token}' + F'{self.bos_token}'.join(lowerCamelCase						) + F'{self.bos_token}Bot:'
      )
      return self.encode(text=lowerCamelCase						)







'''simple docstring'''






from collections.abc import Callable






class   __snake_case			:







    """simple docstring"""



    def __init__(  self       :    Tuple       ,		lowerCamelCase       :    Callable | None = None						)  ->    None:
      # Stores actual heap items.
      lowerCAmelCase_     : list     =  []
      # Stores indexes of each item for supporting updates and deletion.
      lowerCAmelCase_     : dict     =  {}
      # Stores current size of heap.
      lowerCAmelCase_     : List[Any]     =  0
      # Stores function used to evaluate the score of an item on which basis ordering
      # will be done.
      lowerCAmelCase_     : Tuple     =  key or (lambda lowerCamelCase						: x)

    def __lowercase  (  self       :    Optional[Any]       ,		lowerCamelCase       :    int						)  ->    int | None:
      return int((i - 1) / 2						) if i > 0 else None

    def __lowercase  (  self       :    Union[str, Any]       ,		lowerCamelCase       :    int						)  ->    int | None:
      lowerCAmelCase_     : List[str]     =  int(2 * i + 1						)
      return left if 0 < left < self.size else None

    def __lowercase  (  self       :    Optional[Any]       ,		lowerCamelCase       :    int						)  ->    int | None:
      lowerCAmelCase_     : List[Any]     =  int(2 * i + 2						)
      return right if 0 < right < self.size else None

    def __lowercase  (  self       :    List[str]       ,		lowerCamelCase       :    int       ,		lowerCamelCase       :    int						)  ->    None:
      lowerCAmelCase_,				lowerCAmelCase_     : Union[str, Any]     =  (
          self.pos_map[self.arr[j][0]],
          self.pos_map[self.arr[i][0]],
      )
      # Then swap the items in the list.
      lowerCAmelCase_,				lowerCAmelCase_     : List[Any]     =  self.arr[j], self.arr[i]

    def __lowercase  (  self       :    Tuple       ,		lowerCamelCase       :    int       ,		lowerCamelCase       :    int						)  ->    bool:
      return self.arr[i][1] < self.arr[j][1]

    def __lowercase  (  self       :    int       ,		lowerCamelCase       :    int						)  ->    int:
      lowerCAmelCase_     : List[str]     =  self._left(lowerCamelCase						)
      lowerCAmelCase_     : Optional[Any]     =  self._right(lowerCamelCase						)
      lowerCAmelCase_     : Tuple     =  i

      if left is not None and not self._cmp(lowerCamelCase       ,		lowerCamelCase						):
        lowerCAmelCase_     : int     =  left
      if right is not None and not self._cmp(lowerCamelCase       ,		lowerCamelCase						):
        lowerCAmelCase_     : Optional[Any]     =  right

      return valid_parent

    def __lowercase  (  self       :    List[Any]       ,		lowerCamelCase       :    int						)  ->    None:
      lowerCAmelCase_     : Tuple     =  self._parent(lowerCamelCase						)
      while parent is not None and not self._cmp(lowerCamelCase       ,		lowerCamelCase						):
        self._swap(lowerCamelCase       ,		lowerCamelCase						)
        lowerCAmelCase_,				lowerCAmelCase_     : str     =  parent, self._parent(lowerCamelCase						)

    def __lowercase  (  self       :    Tuple       ,		lowerCamelCase       :    int						)  ->    None:
      lowerCAmelCase_     : Optional[Any]     =  self._get_valid_parent(lowerCamelCase						)
      while valid_parent != index:
        self._swap(lowerCamelCase       ,		lowerCamelCase						)
        lowerCAmelCase_,				lowerCAmelCase_     : int     =  valid_parent, self._get_valid_parent(lowerCamelCase						)

    def __lowercase  (  self       :    Optional[Any]       ,		lowerCamelCase       :    int       ,		lowerCamelCase       :    int						)  ->    None:
      if item not in self.pos_map:
        return
      lowerCAmelCase_     : Dict     =  self.pos_map[item]
      lowerCAmelCase_     : Dict     =  [item, self.key(lowerCamelCase						)]
      # Make sure heap is right in both up and down direction.
      # Ideally only one of them will make any change.
      self._heapify_up(lowerCamelCase						)
      self._heapify_down(lowerCamelCase						)

    def __lowercase  (  self       :    int       ,		lowerCamelCase       :    int						)  ->    None:
      if item not in self.pos_map:
        return
      lowerCAmelCase_     : List[str]     =  self.pos_map[item]
      del self.pos_map[item]
      lowerCAmelCase_     : Tuple     =  self.arr[self.size - 1]
      lowerCAmelCase_     : List[str]     =  index
      self.size -= 1
      # Make sure heap is right in both up and down direction. Ideally only one
      # of them will make any change- so no performance loss in calling both.
      if self.size > index:
        self._heapify_up(lowerCamelCase						)
        self._heapify_down(lowerCamelCase						)

    def __lowercase  (  self       :    List[Any]       ,		lowerCamelCase       :    int       ,		lowerCamelCase       :    int						)  ->    None:
      lowerCAmelCase_     : Any     =  len(self.arr						)
      if arr_len == self.size:
        self.arr.append([item, self.key(lowerCamelCase						)]						)
      else:
        lowerCAmelCase_     : str     =  [item, self.key(lowerCamelCase						)]
      lowerCAmelCase_     : Optional[Any]     =  self.size
      self.size += 1
      self._heapify_up(self.size - 1						)

    def __lowercase  (  self       :    str						)  ->    tuple | None:
      return self.arr[0] if self.size else None





    def __lowercase  (  self       :    Optional[Any]						)  ->    tuple | None:
      lowerCAmelCase_     : int     =  self.get_top()
      if top_item_tuple:
        self.delete_item(top_item_tuple[0]						)
      return top_item_tuple







def 							UpperCamelCase_     (       ):
  '''simple docstring'''







if __name__ == "__main__":
  import doctest

  doctest.testmod()



