

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
  import torch


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


lowercase_   							=  logging.get_logger(__name__)
lowercase_   							=  {'vocab_file': 'spiece.model'}

lowercase_   							=  {
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

lowercase_   							=  {
    'AI-Sweden/gpt-sw3-126m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-350m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-1.6b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-6.7b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-20b': 2_0_4_8,
}





class 					__lowerCAmelCase	(   SCREAMING_SNAKE_CASE				):
  _a		     =    VOCAB_FILES_NAMES
  _a		     =    PRETRAINED_VOCAB_FILES_MAP
  _a		     =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _a		     =    ["""input_ids""", """attention_mask"""]




  def __init__(							self				,  lowerCAmelCase				,  lowerCAmelCase=False				,  lowerCAmelCase=False				,  lowerCAmelCase=False				,  lowerCAmelCase=None				,  lowerCAmelCase=None				,  lowerCAmelCase=None				,  lowerCAmelCase=None				,  lowerCAmelCase = None				,  **lowerCAmelCase				,  )    ->					None:
        '''simple docstring'''






        _lowercase      			={} if sp_model_kwargs is None else sp_model_kwargs

        _lowercase      			=kwargs.get('name_or_path'   )
        if name_or_path is None:
              logger.warning(
                  'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
                  ' you are testing the model, this can safely be ignored'   )
              _lowercase      			='None'

        # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
        _lowercase      			='<|endoftext|>' if eos_token is None else eos_token
        _lowercase      			='<unk>' if unk_token is None else unk_token
        if "gpt-sw3-7b" in name_or_path:
              _lowercase      			=unk_token if pad_token is None else pad_token
              _lowercase      			=eos_token if bos_token is None else bos_token
        else:
              _lowercase      			='<pad>' if pad_token is None else pad_token
              _lowercase      			='<s>' if bos_token is None else bos_token

        super().__init__(
            do_lower_case=lowerCAmelCase				,  remove_space=lowerCAmelCase				,  keep_accents=lowerCAmelCase				,  bos_token=lowerCAmelCase				,  eos_token=lowerCAmelCase				,  unk_token=lowerCAmelCase				,  pad_token=lowerCAmelCase				,  sp_model_kwargs=self.sp_model_kwargs				,  **lowerCAmelCase				,  )

        _lowercase      			=do_lower_case
        _lowercase      			=remove_space
        _lowercase      			=keep_accents
        _lowercase      			=vocab_file

        _lowercase      			=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
        self.sp_model.Load(lowerCAmelCase   )

        # Used for whitespace normalization in input texts
        # fmt : off
        _lowercase      			={' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
        # fmt : on

        # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
        _lowercase      			=re.compile(
            F'''[{"".join(map(lowerCAmelCase				,  list(range(0				,  9   )   ) + list(range(11				,  32   )   ) + list(range(127				,  160   )   ) + [160, 173, 8_203]   )   )}]'''   )




  def __getstate__(							self   )    ->					Dict:
        '''simple docstring'''






        _lowercase      			=self.__dict__.copy()
        _lowercase      			=None
        return state




  def __setstate__(							self				,  lowerCAmelCase   )    ->					Dict:
        '''simple docstring'''






        _lowercase      			=d

        # for backward compatibility
        if not hasattr(self				,  'sp_model_kwargs'   ):
              _lowercase      			={}

        _lowercase      			=spm.SentencePieceProcessor(**self.sp_model_kwargs   )
        self.sp_model.Load(self.vocab_file   )




  @property
  # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
  def     A__	(							self   )    ->					int:
        '''simple docstring'''






        return len(self.sp_model   )




  def     A__	(							self				,  lowerCAmelCase   )    ->					str:
        '''simple docstring'''






        _lowercase      			=self.non_printing_characters_re.sub(''				,  lowerCAmelCase   )

        # Normalize whitespaces
        _lowercase      			=''.join([char if char not in self.whitespaces else ' ' for char in text]   )

        # NFC Unicode normalization
        _lowercase      			=unicodedata.normalize('NFC'				,  lowerCAmelCase   )
        return text




  def     A__	(							self				,  lowerCAmelCase				,  **lowerCAmelCase   )    ->					List[str]:
        '''simple docstring'''






        _lowercase      			=self.preprocess_text(lowerCAmelCase   )
        return self.sp_model.encode(lowerCAmelCase				,  out_type=lowerCAmelCase   )




  def     A__	(							self				,  lowerCAmelCase   )    ->					int:
        '''simple docstring'''






        return self.sp_model.PieceToId(lowerCAmelCase   )




  def     A__	(							self				,  lowerCAmelCase   )    ->					str:
        '''simple docstring'''






        return self.sp_model.IdToPiece(lowerCAmelCase   )




  @staticmethod
  def     A__	(							lowerCAmelCase   )    ->					str:
        '''simple docstring'''






        return out_string




  def     A__	(							self				,  lowerCAmelCase   )    ->					str:
        '''simple docstring'''






        _lowercase      			=[]
        _lowercase      			=''
        _lowercase      			=False
        for token in tokens:
              # make sure that special tokens are not decoded using sentencepiece model
              if token in self.all_special_tokens:
                    # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                    if not prev_is_special:
                          out_string += " "

                    out_string += self.sp_model.decode(lowerCAmelCase   ) + token
                    _lowercase      			=True
                    _lowercase      			=[]
              else:
                    current_sub_tokens.append(lowerCAmelCase   )
                    _lowercase      			=False
        out_string += self.sp_model.decode(lowerCAmelCase   )

        return out_string




  def     A__	(							self   )    ->					Dict[str, int]:
        '''simple docstring'''






        _lowercase      			={self.convert_ids_to_tokens(lowerCAmelCase   ): i for i in range(self.vocab_size   )}
        vocab.update(self.added_tokens_encoder   )
        return vocab




  def     A__	(							self				,  lowerCAmelCase				,  lowerCAmelCase = None   )    ->					Tuple[str]:
        '''simple docstring'''






        if not os.path.isdir(lowerCAmelCase   ):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''   )
              return
        _lowercase      			=os.path.join(
            lowerCAmelCase				,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']   )

        if os.path.abspath(self.vocab_file   ) != os.path.abspath(lowerCAmelCase   ) and os.path.isfile(self.vocab_file   ):
              copyfile(self.vocab_file				,  lowerCAmelCase   )
        elif not os.path.isfile(self.vocab_file   ):
              with open(lowerCAmelCase				,  'wb'   ) as fi:
                    _lowercase      			=self.sp_model.serialized_model_proto()
                    fi.write(lowerCAmelCase   )

        return (out_vocab_file,)




  def     A__	(							self				,  lowerCAmelCase				,  lowerCAmelCase = False   )    ->					Union[List[int], List[List[int]], "torch.Tensor"]:
        '''simple docstring'''






        if isinstance(lowerCAmelCase				,  lowerCAmelCase   ):
              _lowercase      			=self.preprocess_text(lowerCAmelCase   )
              _lowercase      			=self.sp_model.encode(lowerCAmelCase   )
        else:
              _lowercase      			=[self.preprocess_text(lowerCAmelCase   ) for t in text]
              _lowercase      			=self.sp_model.encode(lowerCAmelCase   )

        if return_tensors is True or return_tensors == "pt":
              _lowercase      			=torch.tensor(lowerCAmelCase   )

        return token_ids




  def     A__	(							self				,  lowerCAmelCase   )    ->					str:
        '''simple docstring'''






        return self.sp_model.decode(lowerCAmelCase   )




  def     A__	(							self				,  lowerCAmelCase   )    ->					List[int]:
        '''simple docstring'''






        _lowercase      			=[F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
        _lowercase      			=(
            F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(lowerCAmelCase   ) + F'''{self.bos_token}Bot:'''
        )
        return self.encode(text=lowerCAmelCase   )





from collections import OrderedDict
from typing import Any, List, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast, PatchingSpec
from ...utils import logging


lowercase_   							=  logging.get_logger(__name__)

lowercase_   							=  {
    'EleutherAI/gpt-j-6B': 'https://huggingface.co/EleutherAI/gpt-j-6B/resolve/main/config.json',
    # See all GPT-J models at https://huggingface.co/models?filter=gpt_j
}





class 					__lowerCAmelCase	(   SCREAMING_SNAKE_CASE				):
  _a		     =    """gptj"""
  _a		     =    {
      """max_position_embeddings""": """n_positions""",
      """hidden_size""": """n_embd""",
      """num_attention_heads""": """n_head""",
      """num_hidden_layers""": """n_layer""",
  }




  def __init__(							self				,  lowerCAmelCase=50_400				,  lowerCAmelCase=2_048				,  lowerCAmelCase=4_096				,  lowerCAmelCase=28				,  lowerCAmelCase=16				,  lowerCAmelCase=64				,  lowerCAmelCase=None				,  lowerCAmelCase="gelu_new"				,  lowerCAmelCase=0.0				,  lowerCAmelCase=0.0				,  lowerCAmelCase=0.0				,  lowerCAmelCase=1e-5				,  lowerCAmelCase=0.02				,  lowerCAmelCase=True				,  lowerCAmelCase=50_256				,  lowerCAmelCase=50_256				,  lowerCAmelCase=False				,  **lowerCAmelCase				,  )    ->					Optional[Any]:
        '''simple docstring'''






        _lowercase      			=vocab_size
        _lowercase      			=n_positions
        _lowercase      			=n_embd
        _lowercase      			=n_layer
        _lowercase      			=n_head
        _lowercase      			=n_inner
        _lowercase      			=rotary_dim
        _lowercase      			=activation_function
        _lowercase      			=resid_pdrop
        _lowercase      			=embd_pdrop
        _lowercase      			=attn_pdrop
        _lowercase      			=layer_norm_epsilon
        _lowercase      			=initializer_range
        _lowercase      			=use_cache

        _lowercase      			=bos_token_id
        _lowercase      			=eos_token_id

        super().__init__(
            bos_token_id=lowerCAmelCase				,  eos_token_id=lowerCAmelCase				,  tie_word_embeddings=lowerCAmelCase				,  **lowerCAmelCase   )





class 					__lowerCAmelCase	(   SCREAMING_SNAKE_CASE				):




  def __init__(							self				,  lowerCAmelCase				,  lowerCAmelCase = "default"				,  lowerCAmelCase = None				,  lowerCAmelCase = False				,  )    ->					Optional[int]:
        '''simple docstring'''






        super().__init__(lowerCAmelCase				,  task=lowerCAmelCase				,  patching_specs=lowerCAmelCase				,  use_past=lowerCAmelCase   )
        if not getattr(self._config				,  'pad_token_id'				,  lowerCAmelCase   ):
              # TODO: how to do that better?
              _lowercase      			=0




  @property
  def     A__	(							self   )    ->					Mapping[str, Mapping[int, str]]:
        '''simple docstring'''






        _lowercase      			=OrderedDict({'input_ids': {0: 'batch', 1: 'sequence'}}   )
        if self.use_past:
              self.fill_with_past_key_values_(lowerCAmelCase				,  direction='inputs'   )
              _lowercase      			={0: 'batch', 1: 'past_sequence + sequence'}
        else:
              _lowercase      			={0: 'batch', 1: 'sequence'}

        return common_inputs




  @property
  def     A__	(							self   )    ->					int:
        '''simple docstring'''






        return self._config.n_layer




  @property
  def     A__	(							self   )    ->					int:
        '''simple docstring'''






        return self._config.n_head




  def     A__	(							self				,  lowerCAmelCase				,  lowerCAmelCase = -1				,  lowerCAmelCase = -1				,  lowerCAmelCase = False				,  lowerCAmelCase = None				,  )    ->					Mapping[str, Any]:
        '''simple docstring'''






        _lowercase      			=super(lowerCAmelCase				,  self   ).generate_dummy_inputs(
            lowerCAmelCase				,  batch_size=lowerCAmelCase				,  seq_length=lowerCAmelCase				,  is_pair=lowerCAmelCase				,  framework=lowerCAmelCase   )

        # We need to order the input in the way they appears in the forward()
        _lowercase      			=OrderedDict({'input_ids': common_inputs['input_ids']}   )

        # Need to add the past_keys
        if self.use_past:
              if not is_torch_available():
                    raise ValueError('Cannot generate dummy past_keys inputs without PyTorch installed.'   )
              else:
                    import torch

                    _lowercase			,			_lowercase      			=common_inputs['input_ids'].shape
                    # Not using the same length for past_key_values
                    _lowercase      			=seqlen + 2
                    _lowercase      			=(
                        batch,
                        self.num_attention_heads,
                        past_key_values_length,
                        self._config.hidden_size // self.num_attention_heads,
                    )
                    _lowercase      			=[
                        (torch.zeros(lowerCAmelCase   ), torch.zeros(lowerCAmelCase   )) for _ in range(self.num_layers   )
                    ]

        _lowercase      			=common_inputs['attention_mask']
        if self.use_past:
              _lowercase      			=ordered_inputs['attention_mask'].dtype
              _lowercase      			=torch.cat(
                  [ordered_inputs['attention_mask'], torch.ones(lowerCAmelCase				,  lowerCAmelCase				,  dtype=lowerCAmelCase   )]				,  dim=1   )

        return ordered_inputs




  @property
  def     A__	(							self   )    ->					int:
        '''simple docstring'''






        return 13


