




'''simple docstring'''





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


UpperCamelCase_		:     Optional[Any]          =		logging.get_logger(__name__)
UpperCamelCase_		:     Optional[int]          =		{'''vocab_file''': '''spiece.model'''}

UpperCamelCase_		:     int          =		{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

UpperCamelCase_		:     Optional[int]          =		{
    '''AI-Sweden/gpt-sw3-126m''': 2048,
    '''AI-Sweden/gpt-sw3-350m''': 2048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2048,
    '''AI-Sweden/gpt-sw3-20b''': 2048,
}

class   _a							(		__lowerCAmelCase				):
     SCREAMING_SNAKE_CASE_				:       List[str]       							=		VOCAB_FILES_NAMES
     SCREAMING_SNAKE_CASE_				:       Optional[int]       							=		PRETRAINED_VOCAB_FILES_MAP
     SCREAMING_SNAKE_CASE_				:       Any       							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     SCREAMING_SNAKE_CASE_				:       Tuple       							=		["""input_ids""", """attention_mask"""]






     def __init__(					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=False   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE=None   ,_SCREAMING_SNAKE_CASE = None   ,**_SCREAMING_SNAKE_CASE   ,)				->     None:
          _snake_case				= {} if sp_model_kwargs is None else sp_model_kwargs

          _snake_case				= kwargs.get("name_or_path"       )
          if name_or_path is None:
               logger.warning(
                   "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
                   " you are testing the model, this can safely be ignored"       )
               _snake_case				= "None"

          # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
          _snake_case				= "<|endoftext|>" if eos_token is None else eos_token
          _snake_case				= "<unk>" if unk_token is None else unk_token
          if "gpt-sw3-7b" in name_or_path:
               _snake_case				= unk_token if pad_token is None else pad_token
               _snake_case				= eos_token if bos_token is None else bos_token
          else:
               _snake_case				= "<pad>" if pad_token is None else pad_token
               _snake_case				= "<s>" if bos_token is None else bos_token

          super().__init__(
              do_lower_case=_SCREAMING_SNAKE_CASE   ,remove_space=_SCREAMING_SNAKE_CASE   ,keep_accents=_SCREAMING_SNAKE_CASE   ,bos_token=_SCREAMING_SNAKE_CASE   ,eos_token=_SCREAMING_SNAKE_CASE   ,unk_token=_SCREAMING_SNAKE_CASE   ,pad_token=_SCREAMING_SNAKE_CASE   ,sp_model_kwargs=self.sp_model_kwargs   ,**_SCREAMING_SNAKE_CASE   ,)

          _snake_case				= do_lower_case
          _snake_case				= remove_space
          _snake_case				= keep_accents
          _snake_case				= vocab_file

          _snake_case				= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
          self.sp_model.Load(_SCREAMING_SNAKE_CASE       )

          # Used for whitespace normalization in input texts
          # fmt : off
          _snake_case				= {" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
          # fmt : on

          # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
          _snake_case				= re.compile(
              f"""[{''.join(map(_SCREAMING_SNAKE_CASE   ,list(range(0   ,9       )       ) + list(range(11   ,32       )       ) + list(range(127   ,160       )       ) + [160, 173, 8_203]       )       )}]"""       )






     def __getstate__(					self       )				->     Union[str, Any]:
          _snake_case				= self.__dict__.copy()
          _snake_case				= None
          return state






     def __setstate__(					self   ,_SCREAMING_SNAKE_CASE       )				->     Optional[Any]:
          _snake_case				= d

          # for backward compatibility
          if not hasattr(self   ,"sp_model_kwargs"       ):
               _snake_case				= {}

          _snake_case				= spm.SentencePieceProcessor(**self.sp_model_kwargs       )
          self.sp_model.Load(self.vocab_file       )






     @property
     # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
     def 							_lowercase      (					self       )				->     int:
          return len(self.sp_model       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     str:
          _snake_case				= self.non_printing_characters_re.sub(""   ,_SCREAMING_SNAKE_CASE       )

          # Normalize whitespaces
          _snake_case				= "".join([char if char not in self.whitespaces else " " for char in text]       )

          # NFC Unicode normalization
          _snake_case				= unicodedata.normalize("NFC"   ,_SCREAMING_SNAKE_CASE       )
          return text






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,**_SCREAMING_SNAKE_CASE       )				->     List[str]:
          _snake_case				= self.preprocess_text(_SCREAMING_SNAKE_CASE       )
          return self.sp_model.encode(_SCREAMING_SNAKE_CASE   ,out_type=_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     int:
          return self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     str:
          return self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE       )






     @staticmethod
     def 							_lowercase      (					_SCREAMING_SNAKE_CASE       )				->     str:
          return out_string






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     str:
          _snake_case				= []
          _snake_case				= ""
          _snake_case				= False
          for token in tokens:
               # make sure that special tokens are not decoded using sentencepiece model
               if token in self.all_special_tokens:
                    # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                    if not prev_is_special:
                         out_string += " "

                    out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE       ) + token
                    _snake_case				= True
                    _snake_case				= []
               else:
                    current_sub_tokens.append(_SCREAMING_SNAKE_CASE       )
                    _snake_case				= False
          out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE       )

          return out_string






     def 							_lowercase      (					self       )				->     Dict[str, int]:
          _snake_case				= {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE       ): i for i in range(self.vocab_size       )}
          vocab.update(self.added_tokens_encoder       )
          return vocab






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = None       )				->     Tuple[str]:
          if not os.path.isdir(_SCREAMING_SNAKE_CASE       ):
               logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""       )
               return
          _snake_case				= os.path.join(
              _SCREAMING_SNAKE_CASE   ,(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )

          if os.path.abspath(self.vocab_file       ) != os.path.abspath(_SCREAMING_SNAKE_CASE       ) and os.path.isfile(self.vocab_file       ):
               copyfile(self.vocab_file   ,_SCREAMING_SNAKE_CASE       )
          elif not os.path.isfile(self.vocab_file       ):
               with open(_SCREAMING_SNAKE_CASE   ,"wb"       ) as fi:
                    _snake_case				= self.sp_model.serialized_model_proto()
                    fi.write(_SCREAMING_SNAKE_CASE       )

          return (out_vocab_file,)






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE = False       )				->     Union[List[int], List[List[int]], "torch.Tensor"]:

          if isinstance(_SCREAMING_SNAKE_CASE   ,_SCREAMING_SNAKE_CASE       ):
               _snake_case				= self.preprocess_text(_SCREAMING_SNAKE_CASE       )
               _snake_case				= self.sp_model.encode(_SCREAMING_SNAKE_CASE       )
          else:
               _snake_case				= [self.preprocess_text(_SCREAMING_SNAKE_CASE       ) for t in text]
               _snake_case				= self.sp_model.encode(_SCREAMING_SNAKE_CASE       )

          if return_tensors is True or return_tensors == "pt":
               _snake_case				= torch.tensor(_SCREAMING_SNAKE_CASE       )

          return token_ids






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     str:
          return self.sp_model.decode(_SCREAMING_SNAKE_CASE       )






     def 							_lowercase      (					self   ,_SCREAMING_SNAKE_CASE       )				->     List[int]:
          _snake_case				= [f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
          _snake_case				= (
              f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(_SCREAMING_SNAKE_CASE       ) + f"""{self.bos_token}Bot:"""
          )
          return self.encode(text=_SCREAMING_SNAKE_CASE       )








'''simple docstring'''





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


UpperCamelCase_		:     Dict          =		{
    '''configuration_roformer''': ['''ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''RoFormerConfig''', '''RoFormerOnnxConfig'''],
    '''tokenization_roformer''': ['''RoFormerTokenizer'''],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_		:     Union[str, Any]          =		['''RoFormerTokenizerFast''']

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_		:     Tuple          =		[
       '''ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''RoFormerForCausalLM''',
       '''RoFormerForMaskedLM''',
       '''RoFormerForMultipleChoice''',
       '''RoFormerForQuestionAnswering''',
       '''RoFormerForSequenceClassification''',
       '''RoFormerForTokenClassification''',
       '''RoFormerLayer''',
       '''RoFormerModel''',
       '''RoFormerPreTrainedModel''',
       '''load_tf_weights_in_roformer''',
   ]


try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_		:     str          =		[
       '''TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''TFRoFormerForCausalLM''',
       '''TFRoFormerForMaskedLM''',
       '''TFRoFormerForMultipleChoice''',
       '''TFRoFormerForQuestionAnswering''',
       '''TFRoFormerForSequenceClassification''',
       '''TFRoFormerForTokenClassification''',
       '''TFRoFormerLayer''',
       '''TFRoFormerModel''',
       '''TFRoFormerPreTrainedModel''',
   ]


try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   UpperCamelCase_		:     Tuple          =		[
       '''FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST''',
       '''FlaxRoFormerForMaskedLM''',
       '''FlaxRoFormerForMultipleChoice''',
       '''FlaxRoFormerForQuestionAnswering''',
       '''FlaxRoFormerForSequenceClassification''',
       '''FlaxRoFormerForTokenClassification''',
       '''FlaxRoFormerModel''',
       '''FlaxRoFormerPreTrainedModel''',
   ]


if TYPE_CHECKING:
   from .configuration_roformer import ROFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, RoFormerConfig, RoFormerOnnxConfig
   from .tokenization_roformer import RoFormerTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_roformer_fast import RoFormerTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_roformer import (
          ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          RoFormerForCausalLM,
          RoFormerForMaskedLM,
          RoFormerForMultipleChoice,
          RoFormerForQuestionAnswering,
          RoFormerForSequenceClassification,
          RoFormerForTokenClassification,
          RoFormerLayer,
          RoFormerModel,
          RoFormerPreTrainedModel,
          load_tf_weights_in_roformer,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_roformer import (
          TF_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFRoFormerForCausalLM,
          TFRoFormerForMaskedLM,
          TFRoFormerForMultipleChoice,
          TFRoFormerForQuestionAnswering,
          TFRoFormerForSequenceClassification,
          TFRoFormerForTokenClassification,
          TFRoFormerLayer,
          TFRoFormerModel,
          TFRoFormerPreTrainedModel,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_roformer import (
          FLAX_ROFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
          FlaxRoFormerForMaskedLM,
          FlaxRoFormerForMultipleChoice,
          FlaxRoFormerForQuestionAnswering,
          FlaxRoFormerForSequenceClassification,
          FlaxRoFormerForTokenClassification,
          FlaxRoFormerModel,
          FlaxRoFormerPreTrainedModel,
      )


else:
   import sys

   UpperCamelCase_		:     Optional[int]          =		_LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)


