
'''simple docstring'''


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


snake_case_ :  Union[str, Any]     =       logging.get_logger(__name__)
snake_case_ :  Union[str, Any]     =       {'vocab_file': 'spiece.model'}

snake_case_ :  Any     =       {
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

snake_case_ :  Dict     =       {
    'AI-Sweden/gpt-sw3-126m': 2_048,
    'AI-Sweden/gpt-sw3-350m': 2_048,
    'AI-Sweden/gpt-sw3-1.6b': 2_048,
    'AI-Sweden/gpt-sw3-6.7b': 2_048,
    'AI-Sweden/gpt-sw3-20b': 2_048,
}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							['''input_ids''', '''attention_mask''']







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__=False ,   lowerCamelCase__=False ,   lowerCamelCase__=False ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__ = None ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  {} if sp_model_kwargs is None else sp_model_kwargs

      UpperCamelCase       =  kwargs.get('''name_or_path'''   )
      if name_or_path is None:
         logger.warning(
             '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
             ''' you are testing the model, this can safely be ignored'''   )
         UpperCamelCase       =  '''None'''

      # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
      UpperCamelCase       =  '''<|endoftext|>''' if eos_token is None else eos_token
      UpperCamelCase       =  '''<unk>''' if unk_token is None else unk_token
      if "gpt-sw3-7b" in name_or_path:
         UpperCamelCase       =  unk_token if pad_token is None else pad_token
         UpperCamelCase       =  eos_token if bos_token is None else bos_token
      else:
         UpperCamelCase       =  '''<pad>''' if pad_token is None else pad_token
         UpperCamelCase       =  '''<s>''' if bos_token is None else bos_token

      super().__init__(
          do_lower_case=lowerCamelCase__ ,   remove_space=lowerCamelCase__ ,   keep_accents=lowerCamelCase__ ,   bos_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   **lowerCamelCase__ ,   )

      UpperCamelCase       =  do_lower_case
      UpperCamelCase       =  remove_space
      UpperCamelCase       =  keep_accents
      UpperCamelCase       =  vocab_file

      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(lowerCamelCase__   )

      # Used for whitespace normalization in input texts
      # fmt : off
      UpperCamelCase       =  {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
      # fmt : on

      # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
      UpperCamelCase       =  re.compile(
          f'[{"".join(map(lowerCamelCase__ ,   list(range(0 ,   9   )   ) + list(range(1_1 ,   3_2   )   ) + list(range(1_2_7 ,   1_6_0   )   ) + [1_6_0, 1_7_3, 8_2_0_3]   )   )}]'   )







   def __getstate__(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.__dict__.copy()
      UpperCamelCase       =  None
      return state







   def __setstate__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  d

      # for backward compatibility
      if not hasattr(self ,   '''sp_model_kwargs'''   ):
         UpperCamelCase       =  {}

      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(self.vocab_file   )







   @property
   # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return len(self.sp_model   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.non_printing_characters_re.sub('''''' ,   lowerCamelCase__   )

      # Normalize whitespaces
      UpperCamelCase       =  ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]   )

      # NFC Unicode normalization
      UpperCamelCase       =  unicodedata.normalize('''NFC''' ,   lowerCamelCase__   )
      return text







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.preprocess_text(lowerCamelCase__   )
      return self.sp_model.encode(lowerCamelCase__ ,   out_type=lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.PieceToId(lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.IdToPiece(lowerCamelCase__   )







   @staticmethod
   def       UpperCAmelCase				(			lowerCamelCase__   ):

      '''simple docstring'''







      return out_string







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  []
      UpperCamelCase       =  ''''''
      UpperCamelCase       =  False
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
            if not prev_is_special:
               out_string += " "

            out_string += self.sp_model.decode(lowerCamelCase__   ) + token
            UpperCamelCase       =  True
            UpperCamelCase       =  []
         else:
            current_sub_tokens.append(lowerCamelCase__   )
            UpperCamelCase       =  False
      out_string += self.sp_model.decode(lowerCamelCase__   )

      return out_string







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  {self.convert_ids_to_tokens(lowerCamelCase__   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.vocab_file   ):
         with open(lowerCamelCase__ ,   '''wb'''   ) as fi:
            UpperCamelCase       =  self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase__   )

      return (out_vocab_file,)







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if isinstance(lowerCamelCase__ ,   lowerCamelCase__   ):
         UpperCamelCase       =  self.preprocess_text(lowerCamelCase__   )
         UpperCamelCase       =  self.sp_model.encode(lowerCamelCase__   )
      else:
         UpperCamelCase       =  [self.preprocess_text(lowerCamelCase__   ) for t in text]
         UpperCamelCase       =  self.sp_model.encode(lowerCamelCase__   )

      if return_tensors is True or return_tensors == "pt":
         UpperCamelCase       =  torch.tensor(lowerCamelCase__   )

      return token_ids







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.decode(lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  [f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
      UpperCamelCase       =  (
          f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(lowerCamelCase__   ) + f'{self.bos_token}Bot:'
      )
      return self.encode(text=lowerCamelCase__   )







'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tf_available, is_torch_available


snake_case_ :  List[Any]     =					{
    'configuration_data2vec_audio': ['DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP', 'Data2VecAudioConfig'],
    'configuration_data2vec_text': [
        'DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'Data2VecTextConfig',
        'Data2VecTextOnnxConfig',
    ],
    'configuration_data2vec_vision': [
        'DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP',
        'Data2VecVisionConfig',
        'Data2VecVisionOnnxConfig',
    ],
}

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   snake_case_ :  Union[str, Any]     =					[
       'DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST',
       'Data2VecAudioForAudioFrameClassification',
       'Data2VecAudioForCTC',
       'Data2VecAudioForSequenceClassification',
       'Data2VecAudioForXVector',
       'Data2VecAudioModel',
       'Data2VecAudioPreTrainedModel',
   ]
   snake_case_ :  str     =					[
       'DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST',
       'Data2VecTextForCausalLM',
       'Data2VecTextForMaskedLM',
       'Data2VecTextForMultipleChoice',
       'Data2VecTextForQuestionAnswering',
       'Data2VecTextForSequenceClassification',
       'Data2VecTextForTokenClassification',
       'Data2VecTextModel',
       'Data2VecTextPreTrainedModel',
   ]
   snake_case_ :  Optional[int]     =					[
       'DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST',
       'Data2VecVisionForImageClassification',
       'Data2VecVisionForMaskedImageModeling',
       'Data2VecVisionForSemanticSegmentation',
       'Data2VecVisionModel',
       'Data2VecVisionPreTrainedModel',
   ]

if is_tf_available():
   snake_case_ :  List[str]     =					[
       'TFData2VecVisionForImageClassification',
       'TFData2VecVisionForSemanticSegmentation',
       'TFData2VecVisionModel',
       'TFData2VecVisionPreTrainedModel',
   ]

if TYPE_CHECKING:
   from .configuration_dataavec_audio import DATA2VEC_AUDIO_PRETRAINED_CONFIG_ARCHIVE_MAP, DataaVecAudioConfig
   from .configuration_dataavec_text import (
       DATA2VEC_TEXT_PRETRAINED_CONFIG_ARCHIVE_MAP,
       DataaVecTextConfig,
       DataaVecTextOnnxConfig,
   )
   from .configuration_dataavec_vision import (
       DATA2VEC_VISION_PRETRAINED_CONFIG_ARCHIVE_MAP,
       DataaVecVisionConfig,
       DataaVecVisionOnnxConfig,
   )

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_dataavec_audio import (
          DATA2VEC_AUDIO_PRETRAINED_MODEL_ARCHIVE_LIST,
          DataaVecAudioForAudioFrameClassification,
          DataaVecAudioForCTC,
          DataaVecAudioForSequenceClassification,
          DataaVecAudioForXVector,
          DataaVecAudioModel,
          DataaVecAudioPreTrainedModel,
      )
      from .modeling_dataavec_text import (
          DATA2VEC_TEXT_PRETRAINED_MODEL_ARCHIVE_LIST,
          DataaVecTextForCausalLM,
          DataaVecTextForMaskedLM,
          DataaVecTextForMultipleChoice,
          DataaVecTextForQuestionAnswering,
          DataaVecTextForSequenceClassification,
          DataaVecTextForTokenClassification,
          DataaVecTextModel,
          DataaVecTextPreTrainedModel,
      )
      from .modeling_dataavec_vision import (
          DATA2VEC_VISION_PRETRAINED_MODEL_ARCHIVE_LIST,
          DataaVecVisionForImageClassification,
          DataaVecVisionForMaskedImageModeling,
          DataaVecVisionForSemanticSegmentation,
          DataaVecVisionModel,
          DataaVecVisionPreTrainedModel,
      )
   if is_tf_available():
      from .modeling_tf_dataavec_vision import (
          TFDataaVecVisionForImageClassification,
          TFDataaVecVisionForSemanticSegmentation,
          TFDataaVecVisionModel,
          TFDataaVecVisionPreTrainedModel,
      )

else:
   import sys

   snake_case_ :  int     =					_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)





