






'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
      import torch


if TYPE_CHECKING:
      from transformers.pipelines.conversational import Conversation


UpperCamelCase      					=logging.get_logger(__name__)
UpperCamelCase      					={"vocab_file": "spiece.model"}

UpperCamelCase      					={
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

UpperCamelCase      					={
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}




class      A     (   SCREAMING_SNAKE_CASE__    ):






 """simple docstring"""





 __a      :    str			  =    VOCAB_FILES_NAMES
 __a      :    Tuple			  =    PRETRAINED_VOCAB_FILES_MAP
 __a      :    List[Any]			  =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __a      :    int			  =    ['''input_ids''', '''attention_mask''']
 def __init__(				self      ,  __lowerCAmelCase      ,  __lowerCAmelCase=False      ,  __lowerCAmelCase=False      ,  __lowerCAmelCase=False      ,  __lowerCAmelCase=None      ,  __lowerCAmelCase=None      ,  __lowerCAmelCase=None      ,  __lowerCAmelCase=None      ,  __lowerCAmelCase = None      ,  **__lowerCAmelCase      ,  ):
       UpperCamelCase_					: Optional[int]            =				{} if sp_model_kwargs is None else sp_model_kwargs

       UpperCamelCase_					: List[Any]            =				kwargs.get("""name_or_path"""			)
       if name_or_path is None:
             logger.warning(
                 """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
                 """ you are testing the model, this can safely be ignored"""			)
             UpperCamelCase_					: str            =				"""None"""

       # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
       UpperCamelCase_					: Any            =				"""<|endoftext|>""" if eos_token is None else eos_token
       UpperCamelCase_					: Tuple            =				"""<unk>""" if unk_token is None else unk_token
       if "gpt-sw3-7b" in name_or_path:
             UpperCamelCase_					: int            =				unk_token if pad_token is None else pad_token
             UpperCamelCase_					: Dict            =				eos_token if bos_token is None else bos_token
       else:
             UpperCamelCase_					: Optional[int]            =				"""<pad>""" if pad_token is None else pad_token
             UpperCamelCase_					: List[str]            =				"""<s>""" if bos_token is None else bos_token

       super().__init__(
           do_lower_case=__lowerCAmelCase      ,  remove_space=__lowerCAmelCase      ,  keep_accents=__lowerCAmelCase      ,  bos_token=__lowerCAmelCase      ,  eos_token=__lowerCAmelCase      ,  unk_token=__lowerCAmelCase      ,  pad_token=__lowerCAmelCase      ,  sp_model_kwargs=self.sp_model_kwargs      ,  **__lowerCAmelCase      ,  )

       UpperCamelCase_					: Union[str, Any]            =				do_lower_case
       UpperCamelCase_					: Optional[Any]            =				remove_space
       UpperCamelCase_					: Optional[int]            =				keep_accents
       UpperCamelCase_					: List[Any]            =				vocab_file

       UpperCamelCase_					: Optional[Any]            =				spm.SentencePieceProcessor(**self.sp_model_kwargs			)
       self.sp_model.Load(__lowerCAmelCase			)

       # Used for whitespace normalization in input texts
       # fmt : off
       UpperCamelCase_					: List[Any]            =				{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
       # fmt : on

       # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
       UpperCamelCase_					: Any            =				re.compile(
           F"[{''.join(map(__lowerCAmelCase      ,  list(range(0      ,  9			)			) + list(range(11      ,  32			)			) + list(range(1_27      ,  1_60			)			) + [1_60, 1_73, 82_03]			)			)}]"			)
 def __getstate__(				self			):
       UpperCamelCase_					: Optional[int]            =				self.__dict__.copy()
       UpperCamelCase_					: str            =				None
       return state
 def __setstate__(				self      ,  __lowerCAmelCase			):
       UpperCamelCase_					: List[Any]            =				d

       # for backward compatibility
       if not hasattr(self      ,  """sp_model_kwargs"""			):
             UpperCamelCase_					: Tuple            =				{}

       UpperCamelCase_					: Union[str, Any]            =				spm.SentencePieceProcessor(**self.sp_model_kwargs			)
       self.sp_model.Load(self.vocab_file			)
 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 	_UpperCAmelCase							(				self			):
       return len(self.sp_model			)
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase			):
       UpperCamelCase_					: str            =				self.non_printing_characters_re.sub(""""""      ,  __lowerCAmelCase			)

       # Normalize whitespaces
       UpperCamelCase_					: Union[str, Any]            =				"""""".join([char if char not in self.whitespaces else """ """ for char in text]			)

       # NFC Unicode normalization
       UpperCamelCase_					: List[str]            =				unicodedata.normalize("""NFC"""      ,  __lowerCAmelCase			)
       return text
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase      ,  **__lowerCAmelCase			):
       UpperCamelCase_					: List[Any]            =				self.preprocess_text(__lowerCAmelCase			)
       return self.sp_model.encode(__lowerCAmelCase      ,  out_type=__lowerCAmelCase			)
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase			):
       return self.sp_model.PieceToId(__lowerCAmelCase			)
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase			):
       return self.sp_model.IdToPiece(__lowerCAmelCase			)
 @staticmethod
 def 	_UpperCAmelCase							(				__lowerCAmelCase			):
       return out_string
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase			):
       UpperCamelCase_					: int            =				[]
       UpperCamelCase_					: Tuple            =				""""""
       UpperCamelCase_					: Any            =				False
       for token in tokens:
             # make sure that special tokens are not decoded using sentencepiece model
             if token in self.all_special_tokens:
                   # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                   if not prev_is_special:
                         out_string += " "

                   out_string += self.sp_model.decode(__lowerCAmelCase			) + token
                   UpperCamelCase_					: int            =				True
                   UpperCamelCase_					: str            =				[]
             else:
                   current_sub_tokens.append(__lowerCAmelCase			)
                   UpperCamelCase_					: Optional[Any]            =				False
       out_string += self.sp_model.decode(__lowerCAmelCase			)

       return out_string
 def 	_UpperCAmelCase							(				self			):
       UpperCamelCase_					: Dict            =				{self.convert_ids_to_tokens(__lowerCAmelCase			): i for i in range(self.vocab_size			)}
       vocab.update(self.added_tokens_encoder			)
       return vocab
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase      ,  __lowerCAmelCase = None			):
       if not os.path.isdir(__lowerCAmelCase			):
             logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
             return
       UpperCamelCase_					: int            =				os.path.join(
           __lowerCAmelCase      ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

       if os.path.abspath(self.vocab_file			) != os.path.abspath(__lowerCAmelCase			) and os.path.isfile(self.vocab_file			):
             copyfile(self.vocab_file      ,  __lowerCAmelCase			)
       elif not os.path.isfile(self.vocab_file			):
             with open(__lowerCAmelCase      ,  """wb"""			) as fi:
                   UpperCamelCase_					: Optional[Any]            =				self.sp_model.serialized_model_proto()
                   fi.write(__lowerCAmelCase			)

       return (out_vocab_file,)
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase      ,  __lowerCAmelCase = False			):

       if isinstance(__lowerCAmelCase      ,  __lowerCAmelCase			):
             UpperCamelCase_					: Optional[Any]            =				self.preprocess_text(__lowerCAmelCase			)
             UpperCamelCase_					: Any            =				self.sp_model.encode(__lowerCAmelCase			)
       else:
             UpperCamelCase_					: Union[str, Any]            =				[self.preprocess_text(__lowerCAmelCase			) for t in text]
             UpperCamelCase_					: Tuple            =				self.sp_model.encode(__lowerCAmelCase			)

       if return_tensors is True or return_tensors == "pt":
             UpperCamelCase_					: Optional[Any]            =				torch.tensor(__lowerCAmelCase			)

       return token_ids
 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase			):
       return self.sp_model.decode(__lowerCAmelCase			)





 def 	_UpperCAmelCase							(				self      ,  __lowerCAmelCase			):
       UpperCamelCase_					: List[Any]            =				[F"User: {text}" if is_user else F"Bot: {text}" for is_user, text in conversation.iter_texts()]
       UpperCamelCase_					: List[Any]            =				(
           F"{self.eos_token}{self.bos_token}" + F"{self.bos_token}".join(__lowerCAmelCase			) + F"{self.bos_token}Bot:"
       )
       return self.encode(text=__lowerCAmelCase			)














'''simple docstring'''

from transformers import DistilBertTokenizer, DistilBertTokenizerFast
from transformers.testing_utils import require_tokenizers, slow

from ..bert.test_tokenization_bert import BertTokenizationTest




@require_tokenizers
class      A     (   SCREAMING_SNAKE_CASE__    ):






 """simple docstring"""





 __a      :    Optional[int]			  =    DistilBertTokenizer
 __a      :    Union[str, Any]			  =    DistilBertTokenizerFast
 __a      :    Any			  =    True
 @slow
 def 	_UpperCAmelCase							(				self			):
       UpperCamelCase_					: List[Any]            =				DistilBertTokenizer.from_pretrained("""distilbert-base-uncased"""			)

       UpperCamelCase_					: Dict            =				tokenizer.encode("""sequence builders"""      ,  add_special_tokens=__lowerCAmelCase			)
       UpperCamelCase_					: int            =				tokenizer.encode("""multi-sequence build"""      ,  add_special_tokens=__lowerCAmelCase			)

       UpperCamelCase_					: Union[str, Any]            =				tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase			)
       UpperCamelCase_					: Dict            =				tokenizer.build_inputs_with_special_tokens(__lowerCAmelCase      ,  __lowerCAmelCase			)

       assert encoded_sentence == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id]
       assert encoded_pair == [tokenizer.cls_token_id] + text + [tokenizer.sep_token_id] + text_a + [
           tokenizer.sep_token_id
       ]






