






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
  import torch


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE 							=						logging.get_logger(__name__)
SCREAMING_SNAKE_CASE 							=						{'vocab_file': 'spiece.model'}

SCREAMING_SNAKE_CASE 							=						{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

SCREAMING_SNAKE_CASE 							=						{
    'AI-Sweden/gpt-sw3-126m': 2048,
    'AI-Sweden/gpt-sw3-350m': 2048,
    'AI-Sweden/gpt-sw3-1.6b': 2048,
    'AI-Sweden/gpt-sw3-6.7b': 2048,
    'AI-Sweden/gpt-sw3-20b': 2048,
}
class 							A_    (						__lowercase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			Optional[int] =					VOCAB_FILES_NAMES
 _SCREAMING_SNAKE_CASE							:			Optional[Any] =					PRETRAINED_VOCAB_FILES_MAP
 _SCREAMING_SNAKE_CASE							:			Optional[Any] =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					["input_ids", "attention_mask"]


 def __init__(	self							,  _A							,  _A=False							,  _A=False							,  _A=False							,  _A=None							,  _A=None							,  _A=None							,  _A=None							,  _A = None							,  **_A							,  ) ->		None:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				{} if sp_model_kwargs is None else sp_model_kwargs

     _UpperCAmelCase				:      Tuple			  =				kwargs.get('''name_or_path''')
     if name_or_path is None:
         logger.warning(
             '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
             ''' you are testing the model, this can safely be ignored''')
         _UpperCAmelCase				:      List[str]			  =				'''None'''

     # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
     _UpperCAmelCase				:      List[Any]			  =				'''<|endoftext|>''' if eos_token is None else eos_token
     _UpperCAmelCase				:      Union[str, Any]			  =				'''<unk>''' if unk_token is None else unk_token
     if "gpt-sw3-7b" in name_or_path:
         _UpperCAmelCase				:      Optional[Any]			  =				unk_token if pad_token is None else pad_token
         _UpperCAmelCase				:      Optional[Any]			  =				eos_token if bos_token is None else bos_token
     else:
         _UpperCAmelCase				:      List[str]			  =				'''<pad>''' if pad_token is None else pad_token
         _UpperCAmelCase				:      List[Any]			  =				'''<s>''' if bos_token is None else bos_token

     super().__init__(
         do_lower_case=_A							,  remove_space=_A							,  keep_accents=_A							,  bos_token=_A							,  eos_token=_A							,  unk_token=_A							,  pad_token=_A							,  sp_model_kwargs=self.sp_model_kwargs							,  **_A							,  )

     _UpperCAmelCase				:      List[Any]			  =				do_lower_case
     _UpperCAmelCase				:      str			  =				remove_space
     _UpperCAmelCase				:      str			  =				keep_accents
     _UpperCAmelCase				:      Any			  =				vocab_file

     _UpperCAmelCase				:      List[str]			  =				spm.SentencePieceProcessor(**self.sp_model_kwargs)
     self.sp_model.Load(_A)

     # Used for whitespace normalization in input texts
     # fmt : off
     _UpperCAmelCase				:      Optional[int]			  =				{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
     # fmt : on

     # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
     _UpperCAmelCase				:      Optional[int]			  =				re.compile(
         f'''[{''.join(map(_A							,  list(range(0							,  9)) + list(range(11							,  32)) + list(range(127							,  160)) + [160, 173, 8203]))}]''')


 def __getstate__(	self) ->		Union[str, Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Any			  =				self.__dict__.copy()
     _UpperCAmelCase				:      List[Any]			  =				None
     return state


 def __setstate__(	self							,  _A) ->		List[Any]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				d

     # for backward compatibility
     if not hasattr(self							,  '''sp_model_kwargs'''):
         _UpperCAmelCase				:      Optional[int]			  =				{}

     _UpperCAmelCase				:      Optional[int]			  =				spm.SentencePieceProcessor(**self.sp_model_kwargs)
     self.sp_model.Load(self.vocab_file)


 @property
 # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
 def  snake_case__				(	self) ->		int:

     """simple docstring"""



     return len(self.sp_model)


 def  snake_case__				(	self							,  _A) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      List[Any]			  =				self.non_printing_characters_re.sub(''''''							,  _A)

     # Normalize whitespaces
     _UpperCAmelCase				:      str			  =				''''''.join([char if char not in self.whitespaces else ''' ''' for char in text])

     # NFC Unicode normalization
     _UpperCAmelCase				:      Optional[int]			  =				unicodedata.normalize('''NFC'''							,  _A)
     return text


 def  snake_case__				(	self							,  _A							,  **_A) ->		List[str]:

     """simple docstring"""



     _UpperCAmelCase				:      Union[str, Any]			  =				self.preprocess_text(_A)
     return self.sp_model.encode(_A							,  out_type=_A)


 def  snake_case__				(	self							,  _A) ->		int:

     """simple docstring"""



     return self.sp_model.PieceToId(_A)


 def  snake_case__				(	self							,  _A) ->		str:

     """simple docstring"""



     return self.sp_model.IdToPiece(_A)


 @staticmethod
 def  snake_case__				(	_A) ->		str:

     """simple docstring"""



     return out_string


 def  snake_case__				(	self							,  _A) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      List[str]			  =				[]
     _UpperCAmelCase				:      List[Any]			  =				''''''
     _UpperCAmelCase				:      Any			  =				False
     for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
             # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
             if not prev_is_special:
                 out_string += " "

             out_string += self.sp_model.decode(_A) + token
             _UpperCAmelCase				:      Any			  =				True
             _UpperCAmelCase				:      int			  =				[]
         else:
             current_sub_tokens.append(_A)
             _UpperCAmelCase				:      Any			  =				False
     out_string += self.sp_model.decode(_A)

     return out_string


 def  snake_case__				(	self) ->		Dict[str, int]:

     """simple docstring"""



     _UpperCAmelCase				:      int			  =				{self.convert_ids_to_tokens(_A): i for i in range(self.vocab_size)}
     vocab.update(self.added_tokens_encoder)
     return vocab


 def  snake_case__				(	self							,  _A							,  _A = None) ->		Tuple[str]:

     """simple docstring"""



     if not os.path.isdir(_A):
         logger.error(f'''Vocabulary path ({save_directory}) should be a directory''')
         return
     _UpperCAmelCase				:      Optional[int]			  =				os.path.join(
         _A							,  (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

     if os.path.abspath(self.vocab_file) != os.path.abspath(_A) and os.path.isfile(self.vocab_file):
         copyfile(self.vocab_file							,  _A)
     elif not os.path.isfile(self.vocab_file):
         with open(_A							,  '''wb''') as fi:
             _UpperCAmelCase				:      Dict			  =				self.sp_model.serialized_model_proto()
             fi.write(_A)

     return (out_vocab_file,)


 def  snake_case__				(	self							,  _A							,  _A = False) ->		Union[List[int], List[List[int]], "torch.Tensor"]:

     """simple docstring"""



     if isinstance(_A							,  _A):
         _UpperCAmelCase				:      Any			  =				self.preprocess_text(_A)
         _UpperCAmelCase				:      Optional[int]			  =				self.sp_model.encode(_A)
     else:
         _UpperCAmelCase				:      Any			  =				[self.preprocess_text(_A) for t in text]
         _UpperCAmelCase				:      str			  =				self.sp_model.encode(_A)

     if return_tensors is True or return_tensors == "pt":
         _UpperCAmelCase				:      List[Any]			  =				torch.tensor(_A)

     return token_ids


 def  snake_case__				(	self							,  _A) ->		str:

     """simple docstring"""



     return self.sp_model.decode(_A)


 def  snake_case__				(	self							,  _A) ->		List[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				[f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
     _UpperCAmelCase				:      Tuple			  =				(
         f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(_A) + f'''{self.bos_token}Bot:'''
     )
     return self.encode(text=_A)












import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_blenderbot import BlenderbotTokenizer


if TYPE_CHECKING:
  from transformers.pipelines.conversational import Conversation

SCREAMING_SNAKE_CASE 							=						logging.get_logger(__name__)


SCREAMING_SNAKE_CASE 							=						{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
    'tokenizer_config_file': 'tokenizer_config.json',
}

SCREAMING_SNAKE_CASE 							=						{
    'vocab_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json'},
    'merges_file': {'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt'},
    'tokenizer_config_file': {
        'facebook/blenderbot-3B': 'https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json'
    },
}

SCREAMING_SNAKE_CASE 							=						{'facebook/blenderbot-3B': 128}
class 							A_    (						__lowercase			):






 '''simple docstring'''




 _SCREAMING_SNAKE_CASE							:			Tuple =					VOCAB_FILES_NAMES
 _SCREAMING_SNAKE_CASE							:			Optional[Any] =					PRETRAINED_VOCAB_FILES_MAP
 _SCREAMING_SNAKE_CASE							:			Optional[int] =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 _SCREAMING_SNAKE_CASE							:			List[Any] =					["input_ids", "attention_mask"]
 _SCREAMING_SNAKE_CASE							:			Dict =					BlenderbotTokenizer


 def __init__(	self							,  _A=None							,  _A=None							,  _A=None							,  _A="replace"							,  _A="<s>"							,  _A="</s>"							,  _A="</s>"							,  _A="<s>"							,  _A="<unk>"							,  _A="<pad>"							,  _A="<mask>"							,  _A=False							,  _A=True							,  **_A							,  ) ->		str:

     """simple docstring"""



     super().__init__(
         _A							,  _A							,  tokenizer_file=_A							,  errors=_A							,  bos_token=_A							,  eos_token=_A							,  sep_token=_A							,  cls_token=_A							,  unk_token=_A							,  pad_token=_A							,  mask_token=_A							,  add_prefix_space=_A							,  trim_offsets=_A							,  **_A							,  )

     _UpperCAmelCase				:      Any			  =				json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
     if pre_tok_state.get('''add_prefix_space'''							,  _A) != add_prefix_space:
         _UpperCAmelCase				:      Optional[int]			  =				getattr(_A							,  pre_tok_state.pop('''type'''))
         _UpperCAmelCase				:      Optional[int]			  =				add_prefix_space
         _UpperCAmelCase				:      Dict			  =				pre_tok_class(**_A)

     _UpperCAmelCase				:      Dict			  =				add_prefix_space

     _UpperCAmelCase				:      Tuple			  =				'''post_processor'''
     _UpperCAmelCase				:      str			  =				getattr(self.backend_tokenizer							,  _A							,  _A)
     if tokenizer_component_instance:
         _UpperCAmelCase				:      str			  =				json.loads(tokenizer_component_instance.__getstate__())

         # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
         if "sep" in state:
             _UpperCAmelCase				:      str			  =				tuple(state['''sep'''])
         if "cls" in state:
             _UpperCAmelCase				:      List[str]			  =				tuple(state['''cls'''])

         _UpperCAmelCase				:      List[str]			  =				False

         if state.get('''add_prefix_space'''							,  _A) != add_prefix_space:
             _UpperCAmelCase				:      int			  =				add_prefix_space
             _UpperCAmelCase				:      Optional[int]			  =				True

         if state.get('''trim_offsets'''							,  _A) != trim_offsets:
             _UpperCAmelCase				:      List[Any]			  =				trim_offsets
             _UpperCAmelCase				:      Any			  =				True

         if changes_to_apply:
             _UpperCAmelCase				:      Optional[Any]			  =				getattr(_A							,  state.pop('''type'''))
             _UpperCAmelCase				:      Union[str, Any]			  =				component_class(**_A)
             setattr(self.backend_tokenizer							,  _A							,  _A)


 @property
 # Copied from transformers.models.roberta.tokenization_roberta_fast.RobertaTokenizerFast.mask_token with Roberta->Blenderbot, RoBERTa->Blenderbot
 def  snake_case__				(	self) ->		str:

     """simple docstring"""



     if self._mask_token is None:
         if self.verbose:
             logger.error('''Using mask_token, but it is not set yet.''')
         return None
     return str(self._mask_token)


 @mask_token.setter
 def  snake_case__				(	self							,  _A) ->		str:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				AddedToken(_A							,  lstrip=_A							,  rstrip=_A) if isinstance(_A							,  _A) else value
     _UpperCAmelCase				:      Union[str, Any]			  =				value


 def  snake_case__				(	self							,  *_A							,  **_A) ->		BatchEncoding:

     """simple docstring"""



     _UpperCAmelCase				:      List[Any]			  =				kwargs.get('''is_split_into_words'''							,  _A)
     assert self.add_prefix_space or not is_split_into_words, (
         f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
         "to use it with pretokenized inputs."
     )

     return super()._batch_encode_plus(*_A							,  **_A)


 def  snake_case__				(	self							,  *_A							,  **_A) ->		BatchEncoding:

     """simple docstring"""



     _UpperCAmelCase				:      Tuple			  =				kwargs.get('''is_split_into_words'''							,  _A)

     assert self.add_prefix_space or not is_split_into_words, (
         f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
         "to use it with pretokenized inputs."
     )

     return super()._encode_plus(*_A							,  **_A)


 def  snake_case__				(	self							,  _A							,  _A = None) ->		Tuple[str]:

     """simple docstring"""



     _UpperCAmelCase				:      str			  =				self._tokenizer.model.save(_A							,  name=_A)
     return tuple(_A)


 def  snake_case__				(	self							,  _A							,  _A = None) ->		List[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Optional[int]			  =				[self.sep_token_id]
     _UpperCAmelCase				:      str			  =				[self.cls_token_id]

     if token_ids_a is None:
         return len(cls + token_ids_a + sep) * [0]
     return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]


 def  snake_case__				(	self							,  _A							,  _A = None) ->		Optional[Any]:

     """simple docstring"""



     return token_ids_a + [self.eos_token_id]


 def  snake_case__				(	self							,  _A) ->		List[int]:

     """simple docstring"""



     _UpperCAmelCase				:      Dict			  =				[]
     for is_user, text in conversation.iter_texts():
         if is_user:
             # We need to space prefix as it's being done within blenderbot
             inputs.append(''' ''' + text)
         else:
             # Generated responses should contain them already.
             inputs.append(_A)

     _UpperCAmelCase				:      int			  =				'''  '''.join(_A)
     _UpperCAmelCase				:      Optional[int]			  =				self.encode(_A)
     if len(_A) > self.model_max_length:
         _UpperCAmelCase				:      str			  =				input_ids[-self.model_max_length :]
         logger.warning(f'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.''')
     return input_ids




