
'''simple docstring'''


import re
from typing import Callable, List, Optional, Union

import tensorflow as tf


try:
   from tensorflow.keras.optimizers.legacy import Adam
except ImportError:
   from tensorflow.keras.optimizers import Adam


class 						lowercase__	(  tf.keras.optimizers.schedules.LearningRateSchedule   ):






   '''simple docstring'''







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ = 1.0 ,   lowerCamelCase__ = None ,   ):

      '''simple docstring'''







      super().__init__()
      UpperCamelCase       =  initial_learning_rate
      UpperCamelCase       =  warmup_steps
      UpperCamelCase       =  power
      UpperCamelCase       =  decay_schedule_fn
      UpperCamelCase       =  name







   def __call__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      with tf.name_scope(self.name or '''WarmUp'''   ) as name:
         # Implements polynomial warmup. i.e., if global_step < warmup_steps, the
         # learning rate will be `global_step/num_warmup_steps * init_lr`.
         UpperCamelCase       =  tf.cast(lowerCamelCase__ ,   tf.floataa   )
         UpperCamelCase       =  tf.cast(self.warmup_steps ,   tf.floataa   )
         UpperCamelCase       =  global_step_float / warmup_steps_float
         UpperCamelCase       =  self.initial_learning_rate * tf.math.pow(lowerCamelCase__ ,   self.power   )
         return tf.cond(
             global_step_float < warmup_steps_float ,   lambda: warmup_learning_rate ,   lambda: self.decay_schedule_fn(step - self.warmup_steps   ) ,   name=lowerCamelCase__ ,   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return {
          "initial_learning_rate": self.initial_learning_rate,
          "decay_schedule_fn": self.decay_schedule_fn,
          "warmup_steps": self.warmup_steps,
          "power": self.power,
          "name": self.name,
      }




def  __snake_case     (    _UpperCAmelCase					:       float,      _UpperCAmelCase					:       int,      _UpperCAmelCase					:       int,      _UpperCAmelCase					:       float = 0.0,      _UpperCAmelCase					:       float = 0.9,      _UpperCAmelCase					:       float = 0.9_9_9,      _UpperCAmelCase					:       float = 1E-8,      _UpperCAmelCase					:       Optional[float] = None,      _UpperCAmelCase					:       Optional[float] = None,      _UpperCAmelCase					:       float = 0.0,      _UpperCAmelCase					:       float = 1.0,      _UpperCAmelCase					:       Optional[List[str]] = None,      ):
   UpperCamelCase       =  tf.keras.optimizers.schedules.PolynomialDecay(
       initial_learning_rate=_UpperCAmelCase,      decay_steps=num_train_steps - num_warmup_steps,      end_learning_rate=init_lr * min_lr_ratio,      power=_UpperCAmelCase,      )
   if num_warmup_steps:
      UpperCamelCase       =  WarmUp(
          initial_learning_rate=_UpperCAmelCase,      decay_schedule_fn=_UpperCAmelCase,      warmup_steps=_UpperCAmelCase,      )
   if weight_decay_rate > 0.0:
      UpperCamelCase       =  AdamWeightDecay(
          learning_rate=_UpperCAmelCase,      weight_decay_rate=_UpperCAmelCase,      beta_a=_UpperCAmelCase,      beta_a=_UpperCAmelCase,      epsilon=_UpperCAmelCase,      clipnorm=_UpperCAmelCase,      global_clipnorm=_UpperCAmelCase,      exclude_from_weight_decay=['''LayerNorm''', '''layer_norm''', '''bias'''],      include_in_weight_decay=_UpperCAmelCase,      )
   else:
      UpperCamelCase       =  tf.keras.optimizers.Adam(
          learning_rate=_UpperCAmelCase,      beta_a=_UpperCAmelCase,      beta_a=_UpperCAmelCase,      epsilon=_UpperCAmelCase,      clipnorm=_UpperCAmelCase,      global_clipnorm=_UpperCAmelCase,      )
   # We return the optimizer and the LR scheduler in order to better track the
   # evolution of the LR independently of the optimizer.
   return optimizer, lr_schedule


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   def __init__(			self ,   lowerCamelCase__ = 0.001 ,   lowerCamelCase__ = 0.9 ,   lowerCamelCase__ = 0.999 ,   lowerCamelCase__ = 1e-7 ,   lowerCamelCase__ = False ,   lowerCamelCase__ = 0.0 ,   lowerCamelCase__ = None ,   lowerCamelCase__ = None ,   lowerCamelCase__ = "AdamWeightDecay" ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      super().__init__(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   **lowerCamelCase__   )
      UpperCamelCase       =  weight_decay_rate
      UpperCamelCase       =  include_in_weight_decay
      UpperCamelCase       =  exclude_from_weight_decay







   @classmethod
   def       UpperCAmelCase				(			cls ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  {'''WarmUp''': WarmUp}
      return super(lowerCamelCase__ ,   cls   ).from_config(lowerCamelCase__ ,   custom_objects=lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      super(lowerCamelCase__ ,   self   )._prepare_local(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
      UpperCamelCase       =  tf.constant(
          self.weight_decay_rate ,   name='''adam_weight_decay_rate'''   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self._do_use_weight_decay(var.name   )
      if do_decay:
         return var.assign_sub(
             learning_rate * var * apply_state[(var.device, var.dtype.base_dtype)]['''weight_decay_rate'''] ,   use_locking=self._use_locking ,   )
      return tf.no_op()







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__=None ,   **lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase     , UpperCamelCase       =  list(zip(*lowerCamelCase__   )   )
      return super(lowerCamelCase__ ,   self   ).apply_gradients(zip(lowerCamelCase__ ,   lowerCamelCase__   ) ,   name=lowerCamelCase__ ,   **lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   ):

      '''simple docstring'''







      if apply_state is None:
         return self._decayed_lr_t[var_dtype], {}

      UpperCamelCase       =  apply_state or {}
      UpperCamelCase       =  apply_state.get((var_device, var_dtype)   )
      if coefficients is None:
         UpperCamelCase       =  self._fallback_apply_state(lowerCamelCase__ ,   lowerCamelCase__   )
         UpperCamelCase       =  coefficients

      return coefficients["lr_t"], {"apply_state": apply_state}







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__=None   ):

      '''simple docstring'''







      UpperCamelCase     , UpperCamelCase       =  self._get_lr(var.device ,   var.dtype.base_dtype ,   lowerCamelCase__   )
      UpperCamelCase       =  self._decay_weights_op(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
      with tf.control_dependencies([decay]   ):
         return super(lowerCamelCase__ ,   self   )._resource_apply_dense(lowerCamelCase__ ,   lowerCamelCase__ ,   **lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__=None   ):

      '''simple docstring'''







      UpperCamelCase     , UpperCamelCase       =  self._get_lr(var.device ,   var.dtype.base_dtype ,   lowerCamelCase__   )
      UpperCamelCase       =  self._decay_weights_op(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__   )
      with tf.control_dependencies([decay]   ):
         return super(lowerCamelCase__ ,   self   )._resource_apply_sparse(lowerCamelCase__ ,   lowerCamelCase__ ,   lowerCamelCase__ ,   **lowerCamelCase__   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  super().get_config()
      config.update({'''weight_decay_rate''': self.weight_decay_rate}   )
      return config







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if self.weight_decay_rate == 0:
         return False

      if self._include_in_weight_decay:
         for r in self._include_in_weight_decay:
            if re.search(lowerCamelCase__ ,   lowerCamelCase__   ) is not None:
               return True

      if self._exclude_from_weight_decay:
         for r in self._exclude_from_weight_decay:
            if re.search(lowerCamelCase__ ,   lowerCamelCase__   ) is not None:
               return False
      return True




class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   def __init__(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  []
      UpperCamelCase       =  None







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      if self._accum_steps is None:
         UpperCamelCase       =  tf.Variable(
             tf.constant(0 ,   dtype=tf.intaa   ) ,   trainable=lowerCamelCase__ ,   synchronization=tf.VariableSynchronization.ON_READ ,   aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA ,   )

      return self._accum_steps.value()







   @property
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      if not self._gradients:
         raise ValueError('''The accumulator should be called first to initialize the gradients'''   )
      return [gradient.value() if gradient is not None else gradient for gradient in self._gradients]







   def __call__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      if not self._gradients:
         UpperCamelCase       =  self.step  # Create the step variable.
         self._gradients.extend(
             [
                 tf.Variable(
                     tf.zeros_like(lowerCamelCase__   ) ,   trainable=lowerCamelCase__ ,   synchronization=tf.VariableSynchronization.ON_READ ,   aggregation=tf.VariableAggregation.ONLY_FIRST_REPLICA ,   )
                 if gradient is not None
                 else gradient
                 for gradient in gradients
             ]   )
      if len(lowerCamelCase__   ) != len(self._gradients   ):
         raise ValueError(f'Expected {len(self._gradients   )} gradients, but got {len(lowerCamelCase__   )}'   )

      for accum_gradient, gradient in zip(self._gradients ,   lowerCamelCase__   ):
         if accum_gradient is not None and gradient is not None:
            accum_gradient.assign_add(lowerCamelCase__   )

      self._accum_steps.assign_add(1   )







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      if not self._gradients:
         return
      self._accum_steps.assign(0   )
      for gradient in self._gradients:
         if gradient is not None:
            gradient.assign(tf.zeros_like(lowerCamelCase__   )   )







'''simple docstring'''


import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
   import torch


if TYPE_CHECKING:
   from transformers.pipelines.conversational import Conversation


snake_case_ :  Union[str, Any]     =					logging.get_logger(__name__)
snake_case_ :  Union[str, Any]     =					{'vocab_file': 'spiece.model'}

snake_case_ :  Any     =					{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

snake_case_ :  Dict     =					{
    'AI-Sweden/gpt-sw3-126m': 2_048,
    'AI-Sweden/gpt-sw3-350m': 2_048,
    'AI-Sweden/gpt-sw3-1.6b': 2_048,
    'AI-Sweden/gpt-sw3-6.7b': 2_048,
    'AI-Sweden/gpt-sw3-20b': 2_048,
}


class 						lowercase__	(  snake_case_   ):






   '''simple docstring'''







   _snake_case		   =							VOCAB_FILES_NAMES
   _snake_case		   =							PRETRAINED_VOCAB_FILES_MAP
   _snake_case		   =							PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   _snake_case		   =							['''input_ids''', '''attention_mask''']







   def __init__(			self ,   lowerCamelCase__ ,   lowerCamelCase__=False ,   lowerCamelCase__=False ,   lowerCamelCase__=False ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__=None ,   lowerCamelCase__ = None ,   **lowerCamelCase__ ,   ):

      '''simple docstring'''







      UpperCamelCase       =  {} if sp_model_kwargs is None else sp_model_kwargs

      UpperCamelCase       =  kwargs.get('''name_or_path'''   )
      if name_or_path is None:
         logger.warning(
             '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
             ''' you are testing the model, this can safely be ignored'''   )
         UpperCamelCase       =  '''None'''

      # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
      UpperCamelCase       =  '''<|endoftext|>''' if eos_token is None else eos_token
      UpperCamelCase       =  '''<unk>''' if unk_token is None else unk_token
      if "gpt-sw3-7b" in name_or_path:
         UpperCamelCase       =  unk_token if pad_token is None else pad_token
         UpperCamelCase       =  eos_token if bos_token is None else bos_token
      else:
         UpperCamelCase       =  '''<pad>''' if pad_token is None else pad_token
         UpperCamelCase       =  '''<s>''' if bos_token is None else bos_token

      super().__init__(
          do_lower_case=lowerCamelCase__ ,   remove_space=lowerCamelCase__ ,   keep_accents=lowerCamelCase__ ,   bos_token=lowerCamelCase__ ,   eos_token=lowerCamelCase__ ,   unk_token=lowerCamelCase__ ,   pad_token=lowerCamelCase__ ,   sp_model_kwargs=self.sp_model_kwargs ,   **lowerCamelCase__ ,   )

      UpperCamelCase       =  do_lower_case
      UpperCamelCase       =  remove_space
      UpperCamelCase       =  keep_accents
      UpperCamelCase       =  vocab_file

      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(lowerCamelCase__   )

      # Used for whitespace normalization in input texts
      # fmt : off
      UpperCamelCase       =  {''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
      # fmt : on

      # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
      UpperCamelCase       =  re.compile(
          f'[{"".join(map(lowerCamelCase__ ,   list(range(0 ,   9   )   ) + list(range(1_1 ,   3_2   )   ) + list(range(1_2_7 ,   1_6_0   )   ) + [1_6_0, 1_7_3, 8_2_0_3]   )   )}]'   )







   def __getstate__(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  self.__dict__.copy()
      UpperCamelCase       =  None
      return state







   def __setstate__(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  d

      # for backward compatibility
      if not hasattr(self ,   '''sp_model_kwargs'''   ):
         UpperCamelCase       =  {}

      UpperCamelCase       =  spm.SentencePieceProcessor(**self.sp_model_kwargs   )
      self.sp_model.Load(self.vocab_file   )







   @property
   # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      return len(self.sp_model   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.non_printing_characters_re.sub('''''' ,   lowerCamelCase__   )

      # Normalize whitespaces
      UpperCamelCase       =  ''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]   )

      # NFC Unicode normalization
      UpperCamelCase       =  unicodedata.normalize('''NFC''' ,   lowerCamelCase__   )
      return text







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   **lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  self.preprocess_text(lowerCamelCase__   )
      return self.sp_model.encode(lowerCamelCase__ ,   out_type=lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.PieceToId(lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.IdToPiece(lowerCamelCase__   )







   @staticmethod
   def       UpperCAmelCase				(			lowerCamelCase__   ):

      '''simple docstring'''







      return out_string







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  []
      UpperCamelCase       =  ''''''
      UpperCamelCase       =  False
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
            if not prev_is_special:
               out_string += " "

            out_string += self.sp_model.decode(lowerCamelCase__   ) + token
            UpperCamelCase       =  True
            UpperCamelCase       =  []
         else:
            current_sub_tokens.append(lowerCamelCase__   )
            UpperCamelCase       =  False
      out_string += self.sp_model.decode(lowerCamelCase__   )

      return out_string







   def       UpperCAmelCase				(			self   ):

      '''simple docstring'''







      UpperCamelCase       =  {self.convert_ids_to_tokens(lowerCamelCase__   ): i for i in range(self.vocab_size   )}
      vocab.update(self.added_tokens_encoder   )
      return vocab







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = None   ):

      '''simple docstring'''







      if not os.path.isdir(lowerCamelCase__   ):
         logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
         return
      UpperCamelCase       =  os.path.join(
          lowerCamelCase__ ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

      if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCamelCase__   ) and os.path.isfile(self.vocab_file   ):
         copyfile(self.vocab_file ,   lowerCamelCase__   )
      elif not os.path.isfile(self.vocab_file   ):
         with open(lowerCamelCase__ ,   '''wb'''   ) as fi:
            UpperCamelCase       =  self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase__   )

      return (out_vocab_file,)







   def       UpperCAmelCase				(			self ,   lowerCamelCase__ ,   lowerCamelCase__ = False   ):

      '''simple docstring'''







      if isinstance(lowerCamelCase__ ,   lowerCamelCase__   ):
         UpperCamelCase       =  self.preprocess_text(lowerCamelCase__   )
         UpperCamelCase       =  self.sp_model.encode(lowerCamelCase__   )
      else:
         UpperCamelCase       =  [self.preprocess_text(lowerCamelCase__   ) for t in text]
         UpperCamelCase       =  self.sp_model.encode(lowerCamelCase__   )

      if return_tensors is True or return_tensors == "pt":
         UpperCamelCase       =  torch.tensor(lowerCamelCase__   )

      return token_ids







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      return self.sp_model.decode(lowerCamelCase__   )







   def       UpperCAmelCase				(			self ,   lowerCamelCase__   ):

      '''simple docstring'''







      UpperCamelCase       =  [f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
      UpperCamelCase       =  (
          f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(lowerCamelCase__   ) + f'{self.bos_token}Bot:'
      )
      return self.encode(text=lowerCamelCase__   )





