


'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
     import torch


if TYPE_CHECKING:
     from transformers.pipelines.conversational import Conversation


a_    :							Optional[Any]         =					logging.get_logger(__name__)
a_    :							List[str]         =					{"""vocab_file""": """spiece.model"""}

a_    :							Optional[Any]         =					{
    """vocab_file""": {
        """AI-Sweden/gpt-sw3-126m""": """https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-350m""": """https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-1.6b""": """https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-6.7b""": """https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model""",
        """AI-Sweden/gpt-sw3-20b""": """https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model""",
    }
}

a_    :							List[Any]         =					{
    """AI-Sweden/gpt-sw3-126m""": 2048,
    """AI-Sweden/gpt-sw3-350m""": 2048,
    """AI-Sweden/gpt-sw3-1.6b""": 2048,
    """AI-Sweden/gpt-sw3-6.7b""": 2048,
    """AI-Sweden/gpt-sw3-20b""": 2048,
}





class 	snake_case       (       lowercase     ):





  """simple docstring"""




  _lowerCamelCase						       =    VOCAB_FILES_NAMES
  _lowerCamelCase						       =    PRETRAINED_VOCAB_FILES_MAP
  _lowerCamelCase						       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _lowerCamelCase						       =    ["input_ids", "attention_mask"]






  def __init__(       self     ,    UpperCamelCase     ,    UpperCamelCase=False     ,    UpperCamelCase=False     ,    UpperCamelCase=False     ,    UpperCamelCase=None     ,    UpperCamelCase=None     ,    UpperCamelCase=None     ,    UpperCamelCase=None     ,    UpperCamelCase = None     ,    **UpperCamelCase     ,    ):







   """simple docstring"""
   lowerCamelCase_						=    {} if sp_model_kwargs is None else sp_model_kwargs

   lowerCamelCase_						=    kwargs.get("name_or_path"	)
   if name_or_path is None:
    logger.warning(
        "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
        " you are testing the model, this can safely be ignored"	)
    lowerCamelCase_						=    "None"

   # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
   lowerCamelCase_						=    "<|endoftext|>" if eos_token is None else eos_token
   lowerCamelCase_						=    "<unk>" if unk_token is None else unk_token
   if "gpt-sw3-7b" in name_or_path:
    lowerCamelCase_						=    unk_token if pad_token is None else pad_token
    lowerCamelCase_						=    eos_token if bos_token is None else bos_token
   else:
    lowerCamelCase_						=    "<pad>" if pad_token is None else pad_token
    lowerCamelCase_						=    "<s>" if bos_token is None else bos_token

   super().__init__(
       do_lower_case=UpperCamelCase     ,    remove_space=UpperCamelCase     ,    keep_accents=UpperCamelCase     ,    bos_token=UpperCamelCase     ,    eos_token=UpperCamelCase     ,    unk_token=UpperCamelCase     ,    pad_token=UpperCamelCase     ,    sp_model_kwargs=self.sp_model_kwargs     ,    **UpperCamelCase     ,    )

   lowerCamelCase_						=    do_lower_case
   lowerCamelCase_						=    remove_space
   lowerCamelCase_						=    keep_accents
   lowerCamelCase_						=    vocab_file

   lowerCamelCase_						=    spm.SentencePieceProcessor(**self.sp_model_kwargs	)
   self.sp_model.Load(UpperCamelCase	)

   # Used for whitespace normalization in input texts
   # fmt : off
   lowerCamelCase_						=    {" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
   # fmt : on

   # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
   lowerCamelCase_						=    re.compile(
       f'''[{"".join(map(UpperCamelCase     ,    list(range(0     ,    9	)	) + list(range(11     ,    32	)	) + list(range(127     ,    160	)	) + [160, 173, 8203]	)	)}]'''	)






  def __getstate__(       self	):







   """simple docstring"""
   lowerCamelCase_						=    self.__dict__.copy()
   lowerCamelCase_						=    None
   return state






  def __setstate__(       self     ,    UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    d

   # for backward compatibility
   if not hasattr(self     ,    "sp_model_kwargs"	):
    lowerCamelCase_						=    {}

   lowerCamelCase_						=    spm.SentencePieceProcessor(**self.sp_model_kwargs	)
   self.sp_model.Load(self.vocab_file	)






  @property
  # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
  def     snake_case						(       self	):







   """simple docstring"""
   return len(self.sp_model	)






  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    self.non_printing_characters_re.sub(""     ,    UpperCamelCase	)

   # Normalize whitespaces
   lowerCamelCase_						=    "".join([char if char not in self.whitespaces else " " for char in text]	)

   # NFC Unicode normalization
   lowerCamelCase_						=    unicodedata.normalize("NFC"     ,    UpperCamelCase	)
   return text






  def     snake_case						(       self     ,    UpperCamelCase     ,    **UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    self.preprocess_text(UpperCamelCase	)
   return self.sp_model.encode(UpperCamelCase     ,    out_type=UpperCamelCase	)






  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   return self.sp_model.PieceToId(UpperCamelCase	)






  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   return self.sp_model.IdToPiece(UpperCamelCase	)






  @staticmethod
  def     snake_case						(       UpperCamelCase	):







   """simple docstring"""
   return out_string






  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    []
   lowerCamelCase_						=    ""
   lowerCamelCase_						=    False
   for token in tokens:
    # make sure that special tokens are not decoded using sentencepiece model
    if token in self.all_special_tokens:
     # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
     if not prev_is_special:
      out_string += " "

     out_string += self.sp_model.decode(UpperCamelCase	) + token
     lowerCamelCase_						=    True
     lowerCamelCase_						=    []
    else:
     current_sub_tokens.append(UpperCamelCase	)
     lowerCamelCase_						=    False
   out_string += self.sp_model.decode(UpperCamelCase	)

   return out_string






  def     snake_case						(       self	):







   """simple docstring"""
   lowerCamelCase_						=    {self.convert_ids_to_tokens(UpperCamelCase	): i for i in range(self.vocab_size	)}
   vocab.update(self.added_tokens_encoder	)
   return vocab






  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase = None	):







   """simple docstring"""






   if not os.path.isdir(UpperCamelCase	):
    logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
    return
   lowerCamelCase_						=    os.path.join(
       UpperCamelCase     ,    (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)

   if os.path.abspath(self.vocab_file	) != os.path.abspath(UpperCamelCase	) and os.path.isfile(self.vocab_file	):
    copyfile(self.vocab_file     ,    UpperCamelCase	)
   elif not os.path.isfile(self.vocab_file	):
    with open(UpperCamelCase     ,    "wb"	) as fi:
     lowerCamelCase_						=    self.sp_model.serialized_model_proto()
     fi.write(UpperCamelCase	)

   return (out_vocab_file,)






  def     snake_case						(       self     ,    UpperCamelCase     ,    UpperCamelCase = False	):







   """simple docstring"""






   if isinstance(UpperCamelCase     ,    UpperCamelCase	):
    lowerCamelCase_						=    self.preprocess_text(UpperCamelCase	)
    lowerCamelCase_						=    self.sp_model.encode(UpperCamelCase	)
   else:
    lowerCamelCase_						=    [self.preprocess_text(UpperCamelCase	) for t in text]
    lowerCamelCase_						=    self.sp_model.encode(UpperCamelCase	)

   if return_tensors is True or return_tensors == "pt":
    lowerCamelCase_						=    torch.tensor(UpperCamelCase	)

   return token_ids






  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   return self.sp_model.decode(UpperCamelCase	)







  def     snake_case						(       self     ,    UpperCamelCase	):







   """simple docstring"""
   lowerCamelCase_						=    [f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
   lowerCamelCase_						=    (
       f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(UpperCamelCase	) + f'''{self.bos_token}Bot:'''
   )
   return self.encode(text=UpperCamelCase	)






from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase_   							=  {
    'configuration_timesformer': ['TIMESFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP', 'TimesformerConfig'],
}

try:
  if not is_torch_available():
    raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
  pass
else:
  lowercase_   							=  [
      'TIMESFORMER_PRETRAINED_MODEL_ARCHIVE_LIST',
      'TimesformerModel',
      'TimesformerForVideoClassification',
      'TimesformerPreTrainedModel',
  ]

if TYPE_CHECKING:
  from .configuration_timesformer import TIMESFORMER_PRETRAINED_CONFIG_ARCHIVE_MAP, TimesformerConfig

  try:
    if not is_torch_available():
      raise OptionalDependencyNotAvailable()
  except OptionalDependencyNotAvailable:
    pass
  else:
    from .modeling_timesformer import (
        TIMESFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
        TimesformerForVideoClassification,
        TimesformerModel,
        TimesformerPreTrainedModel,
    )

else:
  import sys

  lowercase_   							=  _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)


