


'''simple docstring'''






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
 import torch


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


a_					:    Dict          =    logging.get_logger(__name__)
a_					:    int          =    {"vocab_file": "spiece.model"}

a_					:    Union[str, Any]          =    {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

a_					:    List[str]          =    {
    "AI-Sweden/gpt-sw3-126m": 2_0_4_8,
    "AI-Sweden/gpt-sw3-350m": 2_0_4_8,
    "AI-Sweden/gpt-sw3-1.6b": 2_0_4_8,
    "AI-Sweden/gpt-sw3-6.7b": 2_0_4_8,
    "AI-Sweden/gpt-sw3-20b": 2_0_4_8,
}




class  a     (     _SCREAMING_SNAKE_CASE       ):
 _lowerCAmelCase 					=	VOCAB_FILES_NAMES
 _lowerCAmelCase 					=	PRETRAINED_VOCAB_FILES_MAP
 _lowerCAmelCase 					=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _lowerCAmelCase 					=	["""input_ids""", """attention_mask"""]
 def __init__( self      ,				__magic_name__      ,				__magic_name__=False      ,				__magic_name__=False      ,				__magic_name__=False      ,				__magic_name__=None      ,				__magic_name__=None      ,				__magic_name__=None      ,				__magic_name__=None      ,				__magic_name__ = None      ,				**__magic_name__      ,				)					->							None:
      _a			      =							{} if sp_model_kwargs is None else sp_model_kwargs

      _a			      =							kwargs.get('name_or_path'						)
      if name_or_path is None:
           logger.warning(
               'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
               ' you are testing the model, this can safely be ignored'						)
           _a			      =							'None'

      # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
      _a			      =							'<|endoftext|>' if eos_token is None else eos_token
      _a			      =							'<unk>' if unk_token is None else unk_token
      if "gpt-sw3-7b" in name_or_path:
           _a			      =							unk_token if pad_token is None else pad_token
           _a			      =							eos_token if bos_token is None else bos_token
      else:
           _a			      =							'<pad>' if pad_token is None else pad_token
           _a			      =							'<s>' if bos_token is None else bos_token

      super().__init__(
          do_lower_case=__magic_name__      ,				remove_space=__magic_name__      ,				keep_accents=__magic_name__      ,				bos_token=__magic_name__      ,				eos_token=__magic_name__      ,				unk_token=__magic_name__      ,				pad_token=__magic_name__      ,				sp_model_kwargs=self.sp_model_kwargs      ,				**__magic_name__      ,				)

      _a			      =							do_lower_case
      _a			      =							remove_space
      _a			      =							keep_accents
      _a			      =							vocab_file

      _a			      =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(__magic_name__						)

      # Used for whitespace normalization in input texts
      # fmt : off
      _a			      =							{' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
      # fmt : on

      # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
      _a			      =							re.compile(
          f'[{"".join(map(__magic_name__      ,				list(range(0      ,				9						)						) + list(range(11      ,				32						)						) + list(range(1_27      ,				1_60						)						) + [1_60, 1_73, 82_03]						)						)}]'						)
 def __getstate__( self						)					->							Tuple:
      _a			      =							self.__dict__.copy()
      _a			      =							None
      return state
 def __setstate__( self      ,				__magic_name__						)					->							Optional[Any]:
      _a			      =							d

      # for backward compatibility
      if not hasattr(self      ,				'sp_model_kwargs'						):
           _a			      =							{}

      _a			      =							spm.SentencePieceProcessor(**self.sp_model_kwargs						)
      self.sp_model.Load(self.vocab_file						)
 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def 							__UpperCAmelCase		( self						)					->							int:
      return len(self.sp_model						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							str:
      _a			      =							self.non_printing_characters_re.sub(''      ,				__magic_name__						)

      # Normalize whitespaces
      _a			      =							''.join([char if char not in self.whitespaces else ' ' for char in text]						)

      # NFC Unicode normalization
      _a			      =							unicodedata.normalize('NFC'      ,				__magic_name__						)
      return text
 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				**__magic_name__						)					->							List[str]:
      _a			      =							self.preprocess_text(__magic_name__						)
      return self.sp_model.encode(__magic_name__      ,				out_type=__magic_name__						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							int:
      return self.sp_model.PieceToId(__magic_name__						)
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							str:
      return self.sp_model.IdToPiece(__magic_name__						)
 @staticmethod
 def 							__UpperCAmelCase		( __magic_name__						)					->							str:
      return out_string
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							str:
      _a			      =							[]
      _a			      =							''
      _a			      =							False
      for token in tokens:
           # make sure that special tokens are not decoded using sentencepiece model
           if token in self.all_special_tokens:
                # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                if not prev_is_special:
                     out_string += " "

                out_string += self.sp_model.decode(__magic_name__						) + token
                _a			      =							True
                _a			      =							[]
           else:
                current_sub_tokens.append(__magic_name__						)
                _a			      =							False
      out_string += self.sp_model.decode(__magic_name__						)

      return out_string
 def 							__UpperCAmelCase		( self						)					->							Dict[str, int]:
      _a			      =							{self.convert_ids_to_tokens(__magic_name__						): i for i in range(self.vocab_size						)}
      vocab.update(self.added_tokens_encoder						)
      return vocab
 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__ = None						)					->							Tuple[str]:
      if not os.path.isdir(__magic_name__						):
           logger.error(f'Vocabulary path ({save_directory}) should be a directory'						)
           return
      _a			      =							os.path.join(
          __magic_name__      ,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

      if os.path.abspath(self.vocab_file						) != os.path.abspath(__magic_name__						) and os.path.isfile(self.vocab_file						):
           copyfile(self.vocab_file      ,				__magic_name__						)
      elif not os.path.isfile(self.vocab_file						):
           with open(__magic_name__      ,				'wb'						) as fi:
                _a			      =							self.sp_model.serialized_model_proto()
                fi.write(__magic_name__						)

      return (out_vocab_file,)
 def 							__UpperCAmelCase		( self      ,				__magic_name__      ,				__magic_name__ = False						)					->							Union[List[int], List[List[int]], "torch.Tensor"]:

      if isinstance(__magic_name__      ,				__magic_name__						):
           _a			      =							self.preprocess_text(__magic_name__						)
           _a			      =							self.sp_model.encode(__magic_name__						)
      else:
           _a			      =							[self.preprocess_text(__magic_name__						) for t in text]
           _a			      =							self.sp_model.encode(__magic_name__						)

      if return_tensors is True or return_tensors == "pt":
           _a			      =							torch.tensor(__magic_name__						)

      return token_ids
 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							str:
      return self.sp_model.decode(__magic_name__						)




 def 							__UpperCAmelCase		( self      ,				__magic_name__						)					->							List[int]:
      _a			      =							[f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
      _a			      =							(
          f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(__magic_name__						) + f'{self.bos_token}Bot:'
      )
      return self.encode(text=__magic_name__						)





'''simple docstring'''






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


a_					:    str          =    {
    "configuration_data2vec_audio": ["DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP", "Data2VecAudioConfig"],
    "configuration_data2vec_text": [
        "DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "Data2VecTextConfig",
        "Data2VecTextOnnxConfig",
    ],
    "configuration_data2vec_vision": [
        "DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "Data2VecVisionConfig",
        "Data2VecVisionOnnxConfig",
    ],
}

try:
 if not is_torch_available():
  raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
 pass
else:
 a_					:    List[str]          =    [
     "DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST",
     "Data2VecAudioForAudioFrameClassification",
     "Data2VecAudioForCTC",
     "Data2VecAudioForSequenceClassification",
     "Data2VecAudioForXVector",
     "Data2VecAudioModel",
     "Data2VecAudioPreTrainedModel",
 ]
 a_					:    str          =    [
     "DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST",
     "Data2VecTextForCausalLM",
     "Data2VecTextForMaskedLM",
     "Data2VecTextForMultipleChoice",
     "Data2VecTextForQuestionAnswering",
     "Data2VecTextForSequenceClassification",
     "Data2VecTextForTokenClassification",
     "Data2VecTextModel",
     "Data2VecTextPreTrainedModel",
 ]
 a_					:    Tuple          =    [
     "DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST",
     "Data2VecVisionForImageClassification",
     "Data2VecVisionForMaskedImageModeling",
     "Data2VecVisionForSemanticSegmentation",
     "Data2VecVisionModel",
     "Data2VecVisionPreTrainedModel",
 ]

if is_tf_available():
 a_					:    Dict          =    [
     "TFData2VecVisionForImageClassification",
     "TFData2VecVisionForSemanticSegmentation",
     "TFData2VecVisionModel",
     "TFData2VecVisionPreTrainedModel",
 ]

if TYPE_CHECKING:
 from .configuration_dataavec_audio import DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP, DataaVecAudioConfig
 from .configuration_dataavec_text import (
     DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP,
     DataaVecTextConfig,
     DataaVecTextOnnxConfig,
 )
 from .configuration_dataavec_vision import (
     DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP,
     DataaVecVisionConfig,
     DataaVecVisionOnnxConfig,
 )

 try:
  if not is_torch_available():
   raise OptionalDependencyNotAvailable()
 except OptionalDependencyNotAvailable:
  pass
 else:
  from .modeling_dataavec_audio import (
      DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST,
      DataaVecAudioForAudioFrameClassification,
      DataaVecAudioForCTC,
      DataaVecAudioForSequenceClassification,
      DataaVecAudioForXVector,
      DataaVecAudioModel,
      DataaVecAudioPreTrainedModel,
  )
  from .modeling_dataavec_text import (
      DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
      DataaVecTextForCausalLM,
      DataaVecTextForMaskedLM,
      DataaVecTextForMultipleChoice,
      DataaVecTextForQuestionAnswering,
      DataaVecTextForSequenceClassification,
      DataaVecTextForTokenClassification,
      DataaVecTextModel,
      DataaVecTextPreTrainedModel,
  )
  from .modeling_dataavec_vision import (
      DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST,
      DataaVecVisionForImageClassification,
      DataaVecVisionForMaskedImageModeling,
      DataaVecVisionForSemanticSegmentation,
      DataaVecVisionModel,
      DataaVecVisionPreTrainedModel,
  )
 if is_tf_available():
  from .modeling_tf_dataavec_vision import (
      TFDataaVecVisionForImageClassification,
      TFDataaVecVisionForSemanticSegmentation,
      TFDataaVecVisionModel,
      TFDataaVecVisionPreTrainedModel,
  )

else:
 import sys

 a_					:    List[Any]          =    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)

