





"""simple docstring"""





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
       import torch


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE__   =  logging.get_logger(__name__)
SCREAMING_SNAKE_CASE__   =  {"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE__   =  {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

SCREAMING_SNAKE_CASE__   =  {
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}

class 						lowercase   (   _UpperCAmelCase   ):
      _SCREAMING_SNAKE_CASE							      = VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE							      = PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE							      = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE							      = ['input_ids', 'attention_mask']





      def __init__(			self ,		lowercase ,		lowercase=False ,		lowercase=False ,		lowercase=False ,		lowercase=None ,		lowercase=None ,		lowercase=None ,		lowercase=None ,		lowercase = None ,		**lowercase ,		)						->	None:
            lowerCAmelCase    		=			{} if sp_model_kwargs is None else sp_model_kwargs

            lowerCAmelCase    		=			kwargs.get("""name_or_path"""					)
            if name_or_path is None:
                  logger.warning(
                      """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
                      """ you are testing the model, this can safely be ignored"""					)
                  lowerCAmelCase    		=			"""None"""

            # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
            lowerCAmelCase    		=			"""<|endoftext|>""" if eos_token is None else eos_token
            lowerCAmelCase    		=			"""<unk>""" if unk_token is None else unk_token
            if "gpt-sw3-7b" in name_or_path:
                  lowerCAmelCase    		=			unk_token if pad_token is None else pad_token
                  lowerCAmelCase    		=			eos_token if bos_token is None else bos_token
            else:
                  lowerCAmelCase    		=			"""<pad>""" if pad_token is None else pad_token
                  lowerCAmelCase    		=			"""<s>""" if bos_token is None else bos_token

            super().__init__(
                do_lower_case=lowercase ,		remove_space=lowercase ,		keep_accents=lowercase ,		bos_token=lowercase ,		eos_token=lowercase ,		unk_token=lowercase ,		pad_token=lowercase ,		sp_model_kwargs=self.sp_model_kwargs ,		**lowercase ,		)

            lowerCAmelCase    		=			do_lower_case
            lowerCAmelCase    		=			remove_space
            lowerCAmelCase    		=			keep_accents
            lowerCAmelCase    		=			vocab_file

            lowerCAmelCase    		=			spm.SentencePieceProcessor(**self.sp_model_kwargs					)
            self.sp_model.Load(lowercase					)

            # Used for whitespace normalization in input texts
            # fmt : off
            lowerCAmelCase    		=			{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
            # fmt : on

            # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
            lowerCAmelCase    		=			re.compile(
                f'[{"".join(map(lowercase ,		list(range(0 ,		9					)					) + list(range(11 ,		32					)					) + list(range(127 ,		160					)					) + [160, 173, 8_203]					)					)}]'					)





      def __getstate__(			self					)						->	List[Any]:
            lowerCAmelCase    		=			self.__dict__.copy()
            lowerCAmelCase    		=			None
            return state





      def __setstate__(			self ,		lowercase					)						->	Dict:
            lowerCAmelCase    		=			d

            # for backward compatibility
            if not hasattr(self ,		"""sp_model_kwargs"""					):
                  lowerCAmelCase    		=			{}

            lowerCAmelCase    		=			spm.SentencePieceProcessor(**self.sp_model_kwargs					)
            self.sp_model.Load(self.vocab_file					)





      @property
      # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
      def        _snake_case     (			self					)						->	int:
            return len(self.sp_model					)





      def        _snake_case     (			self ,		lowercase					)						->	str:
            lowerCAmelCase    		=			self.non_printing_characters_re.sub("""""" ,		lowercase					)

            # Normalize whitespaces
            lowerCAmelCase    		=			"""""".join([char if char not in self.whitespaces else """ """ for char in text]					)

            # NFC Unicode normalization
            lowerCAmelCase    		=			unicodedata.normalize("""NFC""" ,		lowercase					)
            return text





      def        _snake_case     (			self ,		lowercase ,		**lowercase					)						->	List[str]:
            lowerCAmelCase    		=			self.preprocess_text(lowercase					)
            return self.sp_model.encode(lowercase ,		out_type=lowercase					)





      def        _snake_case     (			self ,		lowercase					)						->	int:
            return self.sp_model.PieceToId(lowercase					)





      def        _snake_case     (			self ,		lowercase					)						->	str:
            return self.sp_model.IdToPiece(lowercase					)





      @staticmethod
      def        _snake_case     (			lowercase					)						->	str:
            return out_string





      def        _snake_case     (			self ,		lowercase					)						->	str:
            lowerCAmelCase    		=			[]
            lowerCAmelCase    		=			""""""
            lowerCAmelCase    		=			False
            for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                        # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                        if not prev_is_special:
                              out_string += " "

                        out_string += self.sp_model.decode(lowercase					) + token
                        lowerCAmelCase    		=			True
                        lowerCAmelCase    		=			[]
                  else:
                        current_sub_tokens.append(lowercase					)
                        lowerCAmelCase    		=			False
            out_string += self.sp_model.decode(lowercase					)

            return out_string





      def        _snake_case     (			self					)						->	Dict[str, int]:
            lowerCAmelCase    		=			{self.convert_ids_to_tokens(lowercase					): i for i in range(self.vocab_size					)}
            vocab.update(self.added_tokens_encoder					)
            return vocab





      def        _snake_case     (			self ,		lowercase ,		lowercase = None					)						->	Tuple[str]:
            if not os.path.isdir(lowercase					):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory'					)
                  return
            lowerCAmelCase    		=			os.path.join(
                lowercase ,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]					)

            if os.path.abspath(self.vocab_file					) != os.path.abspath(lowercase					) and os.path.isfile(self.vocab_file					):
                  copyfile(self.vocab_file ,		lowercase					)
            elif not os.path.isfile(self.vocab_file					):
                  with open(lowercase ,		"""wb"""					) as fi:
                        lowerCAmelCase    		=			self.sp_model.serialized_model_proto()
                        fi.write(lowercase					)

            return (out_vocab_file,)





      def        _snake_case     (			self ,		lowercase ,		lowercase = False					)						->	Union[List[int], List[List[int]], "torch.Tensor"]:

            if isinstance(lowercase ,		lowercase					):
                  lowerCAmelCase    		=			self.preprocess_text(lowercase					)
                  lowerCAmelCase    		=			self.sp_model.encode(lowercase					)
            else:
                  lowerCAmelCase    		=			[self.preprocess_text(lowercase					) for t in text]
                  lowerCAmelCase    		=			self.sp_model.encode(lowercase					)

            if return_tensors is True or return_tensors == "pt":
                  lowerCAmelCase    		=			torch.tensor(lowercase					)

            return token_ids





      def        _snake_case     (			self ,		lowercase					)						->	str:
            return self.sp_model.decode(lowercase					)





      def        _snake_case     (			self ,		lowercase					)						->	List[int]:
            lowerCAmelCase    		=			[f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
            lowerCAmelCase    		=			(
                f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(lowercase					) + f'{self.bos_token}Bot:'
            )
            return self.encode(text=lowercase					)













"""simple docstring"""





import os
import shutil
from pathlib import Path
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ..utils import ONNX_EXTERNAL_WEIGHTS_NAME, ONNX_WEIGHTS_NAME, is_onnx_available, logging


if is_onnx_available():
       import onnxruntime as ort


SCREAMING_SNAKE_CASE__   =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   =  {
    "tensor(bool)": np.bool_,
    "tensor(int8)": np.inta,
    "tensor(uint8)": np.uinta,
    "tensor(int16)": np.intaa,
    "tensor(uint16)": np.uintaa,
    "tensor(int32)": np.intaa,
    "tensor(uint32)": np.uintaa,
    "tensor(int64)": np.intaa,
    "tensor(uint64)": np.uintaa,
    "tensor(float16)": np.floataa,
    "tensor(float)": np.floataa,
    "tensor(double)": np.floataa,
}

class 						lowercase   :





      def __init__(			self ,		lowercase=None ,		**lowercase					)						->	int:
            logger.info("""`diffusers.OnnxRuntimeModel` is experimental and might change in the future."""					)
            lowerCAmelCase    		=			model
            lowerCAmelCase    		=			kwargs.get("""model_save_dir""" ,		lowercase					)
            lowerCAmelCase    		=			kwargs.get("""latest_model_name""" ,		lowercase					)





      def __call__(			self ,		**lowercase					)						->	List[Any]:
            lowerCAmelCase    		=			{k: np.array(lowercase					) for k, v in kwargs.items()}
            return self.model.run(lowercase ,		lowercase					)





      @staticmethod
      def        _snake_case     (			lowercase ,		lowercase=None ,		lowercase=None					)						->	Optional[Any]:
            if provider is None:
                  logger.info("""No onnxruntime provider specified, using CPUExecutionProvider"""					)
                  lowerCAmelCase    		=			"""CPUExecutionProvider"""

            return ort.InferenceSession(lowercase ,		providers=[provider] ,		sess_options=lowercase					)





      def        _snake_case     (			self ,		lowercase ,		lowercase = None ,		**lowercase					)						->	str:
            lowerCAmelCase    		=			file_name if file_name is not None else ONNX_WEIGHTS_NAME

            lowerCAmelCase    		=			self.model_save_dir.joinpath(self.latest_model_name					)
            lowerCAmelCase    		=			Path(lowercase					).joinpath(lowercase					)
            try:
                  shutil.copyfile(lowercase ,		lowercase					)
            except shutil.SameFileError:
                  pass

            # copy external weights (for models >2GB)
            lowerCAmelCase    		=			self.model_save_dir.joinpath(lowercase					)
            if src_path.exists():
                  lowerCAmelCase    		=			Path(lowercase					).joinpath(lowercase					)
                  try:
                        shutil.copyfile(lowercase ,		lowercase					)
                  except shutil.SameFileError:
                        pass





      def        _snake_case     (			self ,		lowercase ,		**lowercase ,		)						->	str:
            if os.path.isfile(lowercase					):
                  logger.error(f'Provided path ({save_directory}) should be a directory, not a file'					)
                  return

            os.makedirs(lowercase ,		exist_ok=lowercase					)

            # saving model weights/files
            self._save_pretrained(lowercase ,		**lowercase					)





      @classmethod
      def        _snake_case     (			cls ,		lowercase ,		lowercase = None ,		lowercase = None ,		lowercase = False ,		lowercase = None ,		lowercase = None ,		lowercase = None ,		lowercase = None ,		**lowercase ,		)						->	Union[str, Any]:
            lowerCAmelCase    		=			file_name if file_name is not None else ONNX_WEIGHTS_NAME
            # load model from local directory
            if os.path.isdir(lowercase					):
                  lowerCAmelCase    		=			OnnxRuntimeModel.load_model(
                      os.path.join(lowercase ,		lowercase					) ,		provider=lowercase ,		sess_options=lowercase					)
                  lowerCAmelCase    		=			Path(lowercase					)
            # load model from hub
            else:
                  # download model
                  lowerCAmelCase    		=			hf_hub_download(
                      repo_id=lowercase ,		filename=lowercase ,		use_auth_token=lowercase ,		revision=lowercase ,		cache_dir=lowercase ,		force_download=lowercase ,		)
                  lowerCAmelCase    		=			Path(lowercase					).parent
                  lowerCAmelCase    		=			Path(lowercase					).name
                  lowerCAmelCase    		=			OnnxRuntimeModel.load_model(lowercase ,		provider=lowercase ,		sess_options=lowercase					)
            return cls(model=lowercase ,		**lowercase					)





      @classmethod
      def        _snake_case     (			cls ,		lowercase ,		lowercase = True ,		lowercase = None ,		lowercase = None ,		**lowercase ,		)						->	List[str]:
            lowerCAmelCase    		=			None
            if len(str(lowercase					).split("""@"""					)					) == 2:
                  lowerCAmelCase		,						lowerCAmelCase    		=			model_id.split("""@"""					)

            return cls._from_pretrained(
                model_id=lowercase ,		revision=lowercase ,		cache_dir=lowercase ,		force_download=lowercase ,		use_auth_token=lowercase ,		**lowercase ,		)






