




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
							import torch


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


UpperCAmelCase_		:				Optional[Any]									=     logging.get_logger(__name__)
UpperCAmelCase_		:				List[Any]									=     {'''vocab_file''': '''spiece.model'''}

UpperCAmelCase_		:				Optional[Any]									=     {
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

UpperCAmelCase_		:				Optional[Any]									=     {
    '''AI-Sweden/gpt-sw3-126m''': 2048,
    '''AI-Sweden/gpt-sw3-350m''': 2048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2048,
    '''AI-Sweden/gpt-sw3-20b''': 2048,
}




class 		__A      (			SCREAMING_SNAKE_CASE__			):
						UpperCamelCase     						=					VOCAB_FILES_NAMES
						UpperCamelCase     						=					PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase     						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase     						=					["""input_ids""", """attention_mask"""]

						def __init__(		self   :List[str]    , __snake_case   :Any    , __snake_case   :Any=False    , __snake_case   :Dict=False    , __snake_case   :List[str]=False    , __snake_case   :Dict=None    , __snake_case   :str=None    , __snake_case   :List[str]=None    , __snake_case   :str=None    , __snake_case   :Union[str, Any] = None    , **__snake_case   :Tuple    , ):
										'''simple docstring'''

										__magic_name__	:	Any								={} if sp_model_kwargs is None else sp_model_kwargs

										__magic_name__	:	Optional[int]								=kwargs.get("""name_or_path""" )
										if name_or_path is None:
														logger.warning(
														    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
														    """ you are testing the model, this can safely be ignored""" )
														__magic_name__	:	Optional[int]								="""None"""

										# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
										__magic_name__	:	Tuple								="""<|endoftext|>""" if eos_token is None else eos_token
										__magic_name__	:	str								="""<unk>""" if unk_token is None else unk_token
										if "gpt-sw3-7b" in name_or_path:
														__magic_name__	:	str								=unk_token if pad_token is None else pad_token
														__magic_name__	:	Union[str, Any]								=eos_token if bos_token is None else bos_token
										else:
														__magic_name__	:	Tuple								="""<pad>""" if pad_token is None else pad_token
														__magic_name__	:	Tuple								="""<s>""" if bos_token is None else bos_token

										super().__init__(
										    do_lower_case=_lowercase    , remove_space=_lowercase    , keep_accents=_lowercase    , bos_token=_lowercase    , eos_token=_lowercase    , unk_token=_lowercase    , pad_token=_lowercase    , sp_model_kwargs=self.sp_model_kwargs    , **_lowercase    , )

										__magic_name__	:	List[str]								=do_lower_case
										__magic_name__	:	int								=remove_space
										__magic_name__	:	str								=keep_accents
										__magic_name__	:	List[str]								=vocab_file

										__magic_name__	:	Optional[Any]								=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(_lowercase )

										# Used for whitespace normalization in input texts
										# fmt : off
										__magic_name__	:	int								={""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
										# fmt : on

										# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
										__magic_name__	:	Dict								=re.compile(
										    f"[{''.join(map(_lowercase    , list(range(0    , 9 ) ) + list(range(11    , 32 ) ) + list(range(1_27    , 1_60 ) ) + [1_60, 1_73, 82_03] ) )}]" )

						def __getstate__(		self   :Union[str, Any] ):
										'''simple docstring'''

										__magic_name__	:	Union[str, Any]								=self.__dict__.copy()
										__magic_name__	:	List[str]								=None
										return state

						def __setstate__(		self   :Optional[Any]    , __snake_case   :Optional[Any] ):
										'''simple docstring'''

										__magic_name__	:	int								=d

										# for backward compatibility
										if not hasattr(self    , """sp_model_kwargs""" ):
														__magic_name__	:	Any								={}

										__magic_name__	:	Tuple								=spm.SentencePieceProcessor(**self.sp_model_kwargs )
										self.sp_model.Load(self.vocab_file )

						@property
						# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
						def     A__						(		self   :Tuple ):
										'''simple docstring'''

										return len(self.sp_model )

						def     A__						(		self   :str    , __snake_case   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	str								=self.non_printing_characters_re.sub(""""""    , _lowercase )

										# Normalize whitespaces
										__magic_name__	:	Any								="""""".join([char if char not in self.whitespaces else """ """ for char in text] )

										# NFC Unicode normalization
										__magic_name__	:	Optional[int]								=unicodedata.normalize("""NFC"""    , _lowercase )
										return text

						def     A__						(		self   :Optional[int]    , __snake_case   :List[str]    , **__snake_case   :List[str] ):
										'''simple docstring'''

										__magic_name__	:	Optional[Any]								=self.preprocess_text(_lowercase )
										return self.sp_model.encode(_lowercase    , out_type=_lowercase )

						def     A__						(		self   :List[str]    , __snake_case   :int ):
										'''simple docstring'''

										return self.sp_model.PieceToId(_lowercase )

						def     A__						(		self   :Dict    , __snake_case   :Optional[Any] ):
										'''simple docstring'''

										return self.sp_model.IdToPiece(_lowercase )

						@staticmethod
						def     A__						(		__snake_case   :Any ):
										'''simple docstring'''

										return out_string

						def     A__						(		self   :Tuple    , __snake_case   :int ):
										'''simple docstring'''

										__magic_name__	:	str								=[]
										__magic_name__	:	str								=""""""
										__magic_name__	:	Any								=False
										for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																		# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																		if not prev_is_special:
																						out_string += " "

																		out_string += self.sp_model.decode(_lowercase ) + token
																		__magic_name__	:	List[Any]								=True
																		__magic_name__	:	str								=[]
														else:
																		current_sub_tokens.append(_lowercase )
																		__magic_name__	:	List[Any]								=False
										out_string += self.sp_model.decode(_lowercase )

										return out_string

						def     A__						(		self   :Any ):
										'''simple docstring'''

										__magic_name__	:	Dict								={self.convert_ids_to_tokens(_lowercase ): i for i in range(self.vocab_size )}
										vocab.update(self.added_tokens_encoder )
										return vocab

						def     A__						(		self   :int    , __snake_case   :str    , __snake_case   :Optional[Any] = None ):
										'''simple docstring'''

										if not os.path.isdir(_lowercase ):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory" )
														return
										__magic_name__	:	List[Any]								=os.path.join(
										    _lowercase    , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

										if os.path.abspath(self.vocab_file ) != os.path.abspath(_lowercase ) and os.path.isfile(self.vocab_file ):
														copyfile(self.vocab_file    , _lowercase )
										elif not os.path.isfile(self.vocab_file ):
														with open(_lowercase    , """wb""" ) as fi:
																		__magic_name__	:	Union[str, Any]								=self.sp_model.serialized_model_proto()
																		fi.write(_lowercase )

										return (out_vocab_file,)

						def     A__						(		self   :str    , __snake_case   :Tuple    , __snake_case   :Union[str, Any] = False ):
										'''simple docstring'''

										if isinstance(_lowercase    , _lowercase ):
														__magic_name__	:	Optional[int]								=self.preprocess_text(_lowercase )
														__magic_name__	:	int								=self.sp_model.encode(_lowercase )
										else:
														__magic_name__	:	str								=[self.preprocess_text(_lowercase ) for t in text]
														__magic_name__	:	List[Any]								=self.sp_model.encode(_lowercase )

										if return_tensors is True or return_tensors == "pt":
														__magic_name__	:	List[str]								=torch.tensor(_lowercase )

										return token_ids

						def     A__						(		self   :List[str]    , __snake_case   :Any ):
										'''simple docstring'''

										return self.sp_model.decode(_lowercase )


						def     A__						(		self   :Tuple    , __snake_case   :Optional[int] ):
										'''simple docstring'''

										__magic_name__	:	Optional[int]								=[f"User: {text}" if is_user else f"Bot: {text}" for is_user, text in conversation.iter_texts()]
										__magic_name__	:	Dict								=(
										    f"{self.eos_token}{self.bos_token}" + f"{self.bos_token}".join(_lowercase ) + f"{self.bos_token}Bot:"
										)
										return self.encode(text=_lowercase )










from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxSeqaSeqConfigWithPast
from ...utils import logging


UpperCAmelCase_		:				str									=     logging.get_logger(__name__)

UpperCAmelCase_		:				Tuple									=     {
    "t5-small": "https://huggingface.co/t5-small/resolve/main/config.json",
    "t5-base": "https://huggingface.co/t5-base/resolve/main/config.json",
    "t5-large": "https://huggingface.co/t5-large/resolve/main/config.json",
    "t5-3b": "https://huggingface.co/t5-3b/resolve/main/config.json",
    "t5-11b": "https://huggingface.co/t5-11b/resolve/main/config.json",
}




class 		__A      (			UpperCamelCase__			):
						UpperCamelCase     						=					"""t5"""
						UpperCamelCase     						=					["""past_key_values"""]
						UpperCamelCase     						=					{"""hidden_size""": """d_model""", """num_attention_heads""": """num_heads""", """num_hidden_layers""": """num_layers"""}

						def __init__(		self   :Optional[Any]    , __snake_case   :Optional[Any]=3_21_28    , __snake_case   :Optional[int]=5_12    , __snake_case   :str=64    , __snake_case   :int=20_48    , __snake_case   :Union[str, Any]=6    , __snake_case   :Union[str, Any]=None    , __snake_case   :Union[str, Any]=8    , __snake_case   :List[str]=32    , __snake_case   :Dict=1_28    , __snake_case   :List[Any]=0.1    , __snake_case   :int=1E-6    , __snake_case   :List[str]=1.0    , __snake_case   :Optional[int]="relu"    , __snake_case   :List[str]=True    , __snake_case   :Union[str, Any]=True    , __snake_case   :Union[str, Any]=0    , __snake_case   :str=1    , **__snake_case   :int    , ):
										'''simple docstring'''

										__magic_name__	:	str								=vocab_size
										__magic_name__	:	str								=d_model
										__magic_name__	:	Optional[Any]								=d_kv
										__magic_name__	:	int								=d_ff
										__magic_name__	:	Union[str, Any]								=num_layers
										__magic_name__	:	Tuple								=(
										    num_decoder_layers if num_decoder_layers is not None else self.num_layers
										)  # default = symmetry
										__magic_name__	:	List[str]								=num_heads
										__magic_name__	:	Optional[Any]								=relative_attention_num_buckets
										__magic_name__	:	Union[str, Any]								=relative_attention_max_distance
										__magic_name__	:	Dict								=dropout_rate
										__magic_name__	:	Dict								=layer_norm_epsilon
										__magic_name__	:	List[str]								=initializer_factor
										__magic_name__	:	Any								=feed_forward_proj
										__magic_name__	:	Optional[int]								=use_cache

										__magic_name__	:	Tuple								=self.feed_forward_proj.split("""-""" )
										__magic_name__	:	int								=act_info[-1]
										__magic_name__	:	int								=act_info[0] == """gated"""

										if len(__snake_case ) > 1 and act_info[0] != "gated" or len(__snake_case ) > 2:
														raise ValueError(
														    f"`feed_forward_proj`: {feed_forward_proj} is not a valid activation function of the dense layer."
														    """Please make sure `feed_forward_proj` is of the format `gated-{ACT_FN}` or `{ACT_FN}`, e.g. """
														    """'gated-gelu' or 'relu'""" )

										# for backwards compatibility
										if feed_forward_proj == "gated-gelu":
														__magic_name__	:	str								="""gelu_new"""

										super().__init__(
										    pad_token_id=__snake_case    , eos_token_id=__snake_case    , is_encoder_decoder=__snake_case    , **__snake_case    , )




class 		__A      (			UpperCamelCase__			):

						@property
						def     A__						(		self   :List[Any] ):
										'''simple docstring'''

										__magic_name__	:	int								={
										    """input_ids""": {0: """batch""", 1: """encoder_sequence"""},
										    """attention_mask""": {0: """batch""", 1: """encoder_sequence"""},
										}
										if self.use_past:
														__magic_name__	:	List[str]								="""past_encoder_sequence + sequence"""
														__magic_name__	:	List[Any]								={0: """batch"""}
														__magic_name__	:	Union[str, Any]								={0: """batch""", 1: """past_decoder_sequence + sequence"""}
										else:
														__magic_name__	:	int								={0: """batch""", 1: """decoder_sequence"""}
														__magic_name__	:	Tuple								={0: """batch""", 1: """decoder_sequence"""}

										if self.use_past:
														self.fill_with_past_key_values_(__snake_case    , direction="""inputs""" )

										return common_inputs


						@property
						def     A__						(		self   :Union[str, Any] ):
										'''simple docstring'''

										return 13




