






'''simple docstring'''



import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


__UpperCamelCase      :       Dict      			=      logging.get_logger(__name__)
__UpperCamelCase      :       List[str]      			=      {"""vocab_file""": """spiece.model"""}

__UpperCamelCase      :       str      			=      {
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

__UpperCamelCase      :       Tuple      			=      {
    """AI-Sweden/gpt-sw3-126m""": 2048,
    """AI-Sweden/gpt-sw3-350m""": 2048,
    """AI-Sweden/gpt-sw3-1.6b""": 2048,
    """AI-Sweden/gpt-sw3-6.7b""": 2048,
    """AI-Sweden/gpt-sw3-20b""": 2048,
}
class 							__SCREAMING_SNAKE_CASE       (							_UpperCAmelCase     ):
      __a  					=VOCAB_FILES_NAMES
      __a  					=PRETRAINED_VOCAB_FILES_MAP
      __a  					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a  					=["input_ids", "attention_mask"]




      def __init__(  self ,   lowerCamelCase ,   lowerCamelCase=False ,   lowerCamelCase=False ,   lowerCamelCase=False ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase = None ,   **lowerCamelCase ,   )							->Union[str, Any]:



         '''simple docstring'''
         __a				     =    {} if sp_model_kwargs is None else sp_model_kwargs

         __a				     =    kwargs.get('name_or_path'      )
         if name_or_path is None:
            logger.warning(
                'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
                ' you are testing the model, this can safely be ignored'      )
            __a				     =    "None"

         # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
         __a				     =    "<|endoftext|>" if eos_token is None else eos_token
         __a				     =    "<unk>" if unk_token is None else unk_token
         if "gpt-sw3-7b" in name_or_path:
            __a				     =    unk_token if pad_token is None else pad_token
            __a				     =    eos_token if bos_token is None else bos_token
         else:
            __a				     =    "<pad>" if pad_token is None else pad_token
            __a				     =    "<s>" if bos_token is None else bos_token

         super().__init__(
             do_lower_case=lowercase__ ,   remove_space=lowercase__ ,   keep_accents=lowercase__ ,   bos_token=lowercase__ ,   eos_token=lowercase__ ,   unk_token=lowercase__ ,   pad_token=lowercase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   **lowercase__ ,   )

         __a				     =    do_lower_case
         __a				     =    remove_space
         __a				     =    keep_accents
         __a				     =    vocab_file

         __a				     =    spm.SentencePieceProcessor(**self.sp_model_kwargs      )
         self.sp_model.Load(lowercase__      )

         # Used for whitespace normalization in input texts
         # fmt : off
         __a				     =    {" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
         # fmt : on

         # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
         __a				     =    re.compile(
             F"""[{"".join(map(lowercase__ ,   list(range(0 ,   9      )      ) + list(range(11 ,   32      )      ) + list(range(127 ,   160      )      ) + [160, 173, 8203]      )      )}]"""      )




      def __getstate__(  self      )							->Optional[int]:



         '''simple docstring'''
         __a				     =    self.__dict__.copy()
         __a				     =    None
         return state




      def __setstate__(  self ,   lowerCamelCase      )							->List[str]:



         '''simple docstring'''
         __a				     =    d

         # for backward compatibility
         if not hasattr(self ,   'sp_model_kwargs'      ):
            __a				     =    {}

         __a				     =    spm.SentencePieceProcessor(**self.sp_model_kwargs      )
         self.sp_model.Load(self.vocab_file      )




      @property
      # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
      def  __UpperCamelCase							(  self      )							->Optional[Any]:



         '''simple docstring'''
         return len(self.sp_model      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->Any:



         '''simple docstring'''
         __a				     =    self.non_printing_characters_re.sub('' ,   lowercase__      )

         # Normalize whitespaces
         __a				     =    "".join([char if char not in self.whitespaces else ' ' for char in text]      )

         # NFC Unicode normalization
         __a				     =    unicodedata.normalize('NFC' ,   lowercase__      )
         return text




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   **lowerCamelCase      )							->List[Any]:



         '''simple docstring'''
         __a				     =    self.preprocess_text(lowercase__      )
         return self.sp_model.encode(lowercase__ ,   out_type=lowercase__      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->List[Any]:



         '''simple docstring'''
         return self.sp_model.PieceToId(lowercase__      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->Optional[Any]:



         '''simple docstring'''
         return self.sp_model.IdToPiece(lowercase__      )




      @staticmethod
      def  __UpperCamelCase							(  lowerCamelCase      )							->List[Any]:



         '''simple docstring'''
         return out_string




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->List[str]:



         '''simple docstring'''
         __a				     =    []
         __a				     =    ""
         __a				     =    False
         for token in tokens:
            # make sure that special tokens are not decoded using sentencepiece model
            if token in self.all_special_tokens:
               # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
               if not prev_is_special:
                  out_string += " "

               out_string += self.sp_model.decode(lowercase__      ) + token
               __a				     =    True
               __a				     =    []
            else:
               current_sub_tokens.append(lowercase__      )
               __a				     =    False
         out_string += self.sp_model.decode(lowercase__      )

         return out_string




      def  __UpperCamelCase							(  self      )							->Any:



         '''simple docstring'''
         __a				     =    {self.convert_ids_to_tokens(lowercase__      ): i for i in range(self.vocab_size      )}
         vocab.update(self.added_tokens_encoder      )
         return vocab




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = None      )							->Union[str, Any]:



         '''simple docstring'''






         if not os.path.isdir(lowercase__      ):
            logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
            return
         __a				     =    os.path.join(
             lowercase__ ,   (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

         if os.path.abspath(self.vocab_file      ) != os.path.abspath(lowercase__      ) and os.path.isfile(self.vocab_file      ):
            copyfile(self.vocab_file ,   lowercase__      )
         elif not os.path.isfile(self.vocab_file      ):
            with open(lowercase__ ,   'wb'      ) as fi:
               __a				     =    self.sp_model.serialized_model_proto()
               fi.write(lowercase__      )

         return (out_vocab_file,)




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = False      )							->Optional[int]:



         '''simple docstring'''






         if isinstance(lowercase__ ,   lowercase__      ):
            __a				     =    self.preprocess_text(lowercase__      )
            __a				     =    self.sp_model.encode(lowercase__      )
         else:
            __a				     =    [self.preprocess_text(lowercase__      ) for t in text]
            __a				     =    self.sp_model.encode(lowercase__      )

         if return_tensors is True or return_tensors == "pt":
            __a				     =    torch.tensor(lowercase__      )

         return token_ids




      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->Dict:



         '''simple docstring'''
         return self.sp_model.decode(lowercase__      )






      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->str:



         '''simple docstring'''
         __a				     =    [F"""User: {text}""" if is_user else F"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
         __a				     =    (
             F"""{self.eos_token}{self.bos_token}""" + F"""{self.bos_token}""".join(lowercase__      ) + F"""{self.bos_token}Bot:"""
         )
         return self.encode(text=lowercase__      )







'''simple docstring'''



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCamelCase      :       Optional[int]      			=      {
    """configuration_deberta""": ["""DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """DebertaConfig""", """DebertaOnnxConfig"""],
    """tokenization_deberta""": ["""DebertaTokenizer"""],
}

try:
 if not is_tokenizers_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __UpperCamelCase      :       Dict      			=      ["""DebertaTokenizerFast"""]

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __UpperCamelCase      :       Tuple      			=      [
     """DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """DebertaForMaskedLM""",
     """DebertaForQuestionAnswering""",
     """DebertaForSequenceClassification""",
     """DebertaForTokenClassification""",
     """DebertaModel""",
     """DebertaPreTrainedModel""",
 ]

try:
 if not is_tf_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 __UpperCamelCase      :       Optional[Any]      			=      [
     """TF_DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST""",
     """TFDebertaForMaskedLM""",
     """TFDebertaForQuestionAnswering""",
     """TFDebertaForSequenceClassification""",
     """TFDebertaForTokenClassification""",
     """TFDebertaModel""",
     """TFDebertaPreTrainedModel""",
 ]


if TYPE_CHECKING:
 from .configuration_deberta import DEBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, DebertaConfig, DebertaOnnxConfig
 from .tokenization_deberta import DebertaTokenizer

 try:
  if not is_tokenizers_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .tokenization_deberta_fast import DebertaTokenizerFast

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_deberta import (
      DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
      DebertaForMaskedLM,
      DebertaForQuestionAnswering,
      DebertaForSequenceClassification,
      DebertaForTokenClassification,
      DebertaModel,
      DebertaPreTrainedModel,
  )

 try:
  if not is_tf_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_tf_deberta import (
      TF_DEBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
      TFDebertaForMaskedLM,
      TFDebertaForQuestionAnswering,
      TFDebertaForSequenceClassification,
      TFDebertaForTokenClassification,
      TFDebertaModel,
      TFDebertaPreTrainedModel,
  )


else:
 import sys

 __UpperCamelCase      :       List[Any]      			=      _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)