




'''simple docstring'''




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
					import torch


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


lowercase__  							=					logging.get_logger(__name__)
lowercase__  							=					{'''vocab_file''': '''spiece.model'''}

lowercase__  							=					{
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

lowercase__  							=					{
    '''AI-Sweden/gpt-sw3-126m''': 20_48,
    '''AI-Sweden/gpt-sw3-350m''': 20_48,
    '''AI-Sweden/gpt-sw3-1.6b''': 20_48,
    '''AI-Sweden/gpt-sw3-6.7b''': 20_48,
    '''AI-Sweden/gpt-sw3-20b''': 20_48,
}






class      UpperCAmelCase_     (  SCREAMING_SNAKE_CASE__     ):


	"""simple docstring"""


	snake_case									=		VOCAB_FILES_NAMES
	snake_case									=		PRETRAINED_VOCAB_FILES_MAP
	snake_case									=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	snake_case									=		["""input_ids""", """attention_mask"""]







	def __init__(						self						, UpperCAmelCase_						, UpperCAmelCase_=False						, UpperCAmelCase_=False						, UpperCAmelCase_=False						, UpperCAmelCase_=None						, UpperCAmelCase_=None						, UpperCAmelCase_=None						, UpperCAmelCase_=None						, UpperCAmelCase_ = None						, **UpperCAmelCase_						, ):
						snake_case_	=      {} if sp_model_kwargs is None else sp_model_kwargs

						snake_case_	=      kwargs.get("name_or_path"				)
						if name_or_path is None:
											logger.warning(
											    "name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"
											    " you are testing the model, this can safely be ignored"				)
											snake_case_	=      "None"

						# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
						snake_case_	=      "<|endoftext|>" if eos_token is None else eos_token
						snake_case_	=      "<unk>" if unk_token is None else unk_token
						if "gpt-sw3-7b" in name_or_path:
											snake_case_	=      unk_token if pad_token is None else pad_token
											snake_case_	=      eos_token if bos_token is None else bos_token
						else:
											snake_case_	=      "<pad>" if pad_token is None else pad_token
											snake_case_	=      "<s>" if bos_token is None else bos_token

						super().__init__(
						    do_lower_case=_lowercase						, remove_space=_lowercase						, keep_accents=_lowercase						, bos_token=_lowercase						, eos_token=_lowercase						, unk_token=_lowercase						, pad_token=_lowercase						, sp_model_kwargs=self.sp_model_kwargs						, **_lowercase						, )

						snake_case_	=      do_lower_case
						snake_case_	=      remove_space
						snake_case_	=      keep_accents
						snake_case_	=      vocab_file

						snake_case_	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
						self.sp_model.Load(_lowercase				)

						# Used for whitespace normalization in input texts
						# fmt : off
						snake_case_	=      {" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
						# fmt : on

						# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
						snake_case_	=      re.compile(
						    f'''[{"".join(map(_lowercase						, list(range(0						, 9				)				) + list(range(11						, 32				)				) + list(range(1_27						, 1_60				)				) + [1_60, 1_73, 82_03]				)				)}]'''				)







	def __getstate__(						self				):
						snake_case_	=      self.__dict__.copy()
						snake_case_	=      None
						return state







	def __setstate__(						self						, UpperCAmelCase_				):
						snake_case_	=      d

						# for backward compatibility
						if not hasattr(self						, "sp_model_kwargs"				):
											snake_case_	=      {}

						snake_case_	=      spm.SentencePieceProcessor(**self.sp_model_kwargs				)
						self.sp_model.Load(self.vocab_file				)







	@property
	# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
	def  _lowercase    (						self				):
						return len(self.sp_model				)







	def  _lowercase    (						self						, UpperCAmelCase_				):
						snake_case_	=      self.non_printing_characters_re.sub(""						, _lowercase				)

						# Normalize whitespaces
						snake_case_	=      "".join([char if char not in self.whitespaces else " " for char in text]				)

						# NFC Unicode normalization
						snake_case_	=      unicodedata.normalize("NFC"						, _lowercase				)
						return text







	def  _lowercase    (						self						, UpperCAmelCase_						, **UpperCAmelCase_				):
						snake_case_	=      self.preprocess_text(_lowercase				)
						return self.sp_model.encode(_lowercase						, out_type=_lowercase				)







	def  _lowercase    (						self						, UpperCAmelCase_				):
						return self.sp_model.PieceToId(_lowercase				)







	def  _lowercase    (						self						, UpperCAmelCase_				):
						return self.sp_model.IdToPiece(_lowercase				)







	@staticmethod
	def  _lowercase    (						UpperCAmelCase_				):
						return out_string







	def  _lowercase    (						self						, UpperCAmelCase_				):
						snake_case_	=      []
						snake_case_	=      ""
						snake_case_	=      False
						for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
																# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																if not prev_is_special:
																					out_string += " "

																out_string += self.sp_model.decode(_lowercase				) + token
																snake_case_	=      True
																snake_case_	=      []
											else:
																current_sub_tokens.append(_lowercase				)
																snake_case_	=      False
						out_string += self.sp_model.decode(_lowercase				)

						return out_string







	def  _lowercase    (						self				):
						snake_case_	=      {self.convert_ids_to_tokens(_lowercase				): i for i in range(self.vocab_size				)}
						vocab.update(self.added_tokens_encoder				)
						return vocab







	def  _lowercase    (						self						, UpperCAmelCase_						, UpperCAmelCase_ = None				):
						if not os.path.isdir(_lowercase				):
											logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''				)
											return
						snake_case_	=      os.path.join(
						    _lowercase						, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

						if os.path.abspath(self.vocab_file				) != os.path.abspath(_lowercase				) and os.path.isfile(self.vocab_file				):
											copyfile(self.vocab_file						, _lowercase				)
						elif not os.path.isfile(self.vocab_file				):
											with open(_lowercase						, "wb"				) as fi:
																snake_case_	=      self.sp_model.serialized_model_proto()
																fi.write(_lowercase				)

						return (out_vocab_file,)







	def  _lowercase    (						self						, UpperCAmelCase_						, UpperCAmelCase_ = False				):

						if isinstance(_lowercase						, _lowercase				):
											snake_case_	=      self.preprocess_text(_lowercase				)
											snake_case_	=      self.sp_model.encode(_lowercase				)
						else:
											snake_case_	=      [self.preprocess_text(_lowercase				) for t in text]
											snake_case_	=      self.sp_model.encode(_lowercase				)

						if return_tensors is True or return_tensors == "pt":
											snake_case_	=      torch.tensor(_lowercase				)

						return token_ids







	def  _lowercase    (						self						, UpperCAmelCase_				):
						return self.sp_model.decode(_lowercase				)







	def  _lowercase    (						self						, UpperCAmelCase_				):
						snake_case_	=      [f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
						snake_case_	=      (
						    f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(_lowercase				) + f'''{self.bos_token}Bot:'''
						)
						return self.encode(text=_lowercase				)











'''simple docstring'''
import os
from typing import List, Optional, Union

from ...image_processing_utils import BatchFeature
from ...image_utils import ImageInput
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import PaddingStrategy, PreTokenizedInput, TextInput, TruncationStrategy
from ...utils import TensorType
from ..auto import AutoTokenizer
class   __UpperCAmelCase				(     lowerCAmelCase				):



					'''simple docstring'''





					_UpperCamelCase							=		["""image_processor""", """tokenizer"""]
					_UpperCamelCase							=		"""BlipImageProcessor"""
					_UpperCamelCase							=		"""AutoTokenizer"""



					def __init__(	self     :						Union[str, Any]      ,       _lowercase     :						Union[str, Any]      ,       _lowercase     :						List[Any]      ,       _lowercase     :						List[str])    ->							Union[str, Any]:
								super().__init__(_lowercase      ,       _lowercase)

								# add QFormer tokenizer
								A_									= qformer_tokenizer



					def __call__(	self     :						Optional[int]      ,       _lowercase     :						ImageInput = None      ,       _lowercase     :						Union[TextInput, PreTokenizedInput, List[TextInput], List[PreTokenizedInput]] = None      ,       _lowercase     :						bool = True      ,       _lowercase     :						Union[bool, str, PaddingStrategy] = False      ,       _lowercase     :						Union[bool, str, TruncationStrategy] = None      ,       _lowercase     :						Optional[int] = None      ,       _lowercase     :						int = 0      ,       _lowercase     :						Optional[int] = None      ,       _lowercase     :						Optional[bool] = None      ,       _lowercase     :						bool = False      ,       _lowercase     :						bool = False      ,       _lowercase     :						bool = False      ,       _lowercase     :						bool = False      ,       _lowercase     :						bool = False      ,       _lowercase     :						bool = True      ,       _lowercase     :						Optional[Union[str, TensorType]] = None      ,       **_lowercase     :						List[Any]      ,       )    ->							BatchFeature:
								if images is None and text is None:
											raise ValueError('You have to specify at least images or text.')

								A_									= BatchFeature()

								if text is not None:
											A_									= self.tokenizer(
											    text=_lowercase      ,       add_special_tokens=_lowercase      ,       padding=_lowercase      ,       truncation=_lowercase      ,       max_length=_lowercase      ,       stride=_lowercase      ,       pad_to_multiple_of=_lowercase      ,       return_attention_mask=_lowercase      ,       return_overflowing_tokens=_lowercase      ,       return_special_tokens_mask=_lowercase      ,       return_offsets_mapping=_lowercase      ,       return_token_type_ids=_lowercase      ,       return_length=_lowercase      ,       verbose=_lowercase      ,       return_tensors=_lowercase      ,       **_lowercase      ,       )
											encoding.update(_lowercase)
											A_									= self.qformer_tokenizer(
											    text=_lowercase      ,       add_special_tokens=_lowercase      ,       padding=_lowercase      ,       truncation=_lowercase      ,       max_length=_lowercase      ,       stride=_lowercase      ,       pad_to_multiple_of=_lowercase      ,       return_attention_mask=_lowercase      ,       return_overflowing_tokens=_lowercase      ,       return_special_tokens_mask=_lowercase      ,       return_offsets_mapping=_lowercase      ,       return_token_type_ids=_lowercase      ,       return_length=_lowercase      ,       verbose=_lowercase      ,       return_tensors=_lowercase      ,       **_lowercase      ,       )
											A_									= qformer_text_encoding.pop('input_ids')
											A_									= qformer_text_encoding.pop('attention_mask')

								if images is not None:
											A_									= self.image_processor(_lowercase      ,       return_tensors=_lowercase)
											encoding.update(_lowercase)

								return encoding



					def     __snake_case						(	self     :						List[Any]      ,       *_lowercase     :						Optional[Any]      ,       **_lowercase     :						List[Any])    ->							Union[str, Any]:
								return self.tokenizer.batch_decode(*_lowercase      ,       **_lowercase)



					def     __snake_case						(	self     :						Optional[int]      ,       *_lowercase     :						List[Any]      ,       **_lowercase     :						List[str])    ->							str:
								return self.tokenizer.decode(*_lowercase      ,       **_lowercase)



					@property
					# Copied from transformers.models.blip.processing_blip.BlipProcessor.model_input_names
					def     __snake_case						(	self     :						Any)    ->							Optional[int]:
								A_									= self.tokenizer.model_input_names
								A_									= self.image_processor.model_input_names
								return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names))



					def     __snake_case						(	self     :						Optional[Any]      ,       _lowercase     :						Optional[int]      ,       **_lowercase     :						int)    ->							Tuple:
								if os.path.isfile(_lowercase):
											raise ValueError(F'Provided path ({save_directory}) should be a directory, not a file')
								os.makedirs(_lowercase      ,       exist_ok=_lowercase)
								A_									= os.path.join(_lowercase      ,       'qformer_tokenizer')
								self.qformer_tokenizer.save_pretrained(_lowercase)
								return super().save_pretrained(_lowercase      ,       **_lowercase)



					@classmethod
					def     __snake_case						(	cls     :						Dict      ,       _lowercase     :						Dict      ,       **_lowercase     :						int)    ->							Any:
								A_									= AutoTokenizer.from_pretrained(_lowercase      ,       subfolder='qformer_tokenizer')
								A_									= cls._get_arguments_from_pretrained(_lowercase      ,       **_lowercase)
								args.append(_lowercase)
								return cls(*_lowercase)




