




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
				import torch


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


a													=			logging.get_logger(__name__)
a													=			{"vocab_file": "spiece.model"}

a													=			{
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

a													=			{
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}




class       _A				(		__lowercase     ):
		__a					    =       VOCAB_FILES_NAMES
		__a					    =       PRETRAINED_VOCAB_FILES_MAP
		__a					    =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__a					    =       ["""input_ids""", """attention_mask"""]






		def __init__(    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE=False						,		_SCREAMING_SNAKE_CASE=False						,		_SCREAMING_SNAKE_CASE=False						,		_SCREAMING_SNAKE_CASE=None						,		_SCREAMING_SNAKE_CASE=None						,		_SCREAMING_SNAKE_CASE=None						,		_SCREAMING_SNAKE_CASE=None						,		_SCREAMING_SNAKE_CASE = None						,		**_SCREAMING_SNAKE_CASE						,		):
								_UpperCAmelCase      =			{} if sp_model_kwargs is None else sp_model_kwargs

								_UpperCAmelCase      =			kwargs.get("""name_or_path"""			)
								if name_or_path is None:
														logger.warning(
														    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
														    """ you are testing the model, this can safely be ignored"""			)
														_UpperCAmelCase      =			"""None"""

								# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
								_UpperCAmelCase      =			"""<|endoftext|>""" if eos_token is None else eos_token
								_UpperCAmelCase      =			"""<unk>""" if unk_token is None else unk_token
								if "gpt-sw3-7b" in name_or_path:
														_UpperCAmelCase      =			unk_token if pad_token is None else pad_token
														_UpperCAmelCase      =			eos_token if bos_token is None else bos_token
								else:
														_UpperCAmelCase      =			"""<pad>""" if pad_token is None else pad_token
														_UpperCAmelCase      =			"""<s>""" if bos_token is None else bos_token

								super().__init__(
								    do_lower_case=_SCREAMING_SNAKE_CASE						,		remove_space=_SCREAMING_SNAKE_CASE						,		keep_accents=_SCREAMING_SNAKE_CASE						,		bos_token=_SCREAMING_SNAKE_CASE						,		eos_token=_SCREAMING_SNAKE_CASE						,		unk_token=_SCREAMING_SNAKE_CASE						,		pad_token=_SCREAMING_SNAKE_CASE						,		sp_model_kwargs=self.sp_model_kwargs						,		**_SCREAMING_SNAKE_CASE						,		)

								_UpperCAmelCase      =			do_lower_case
								_UpperCAmelCase      =			remove_space
								_UpperCAmelCase      =			keep_accents
								_UpperCAmelCase      =			vocab_file

								_UpperCAmelCase      =			spm.SentencePieceProcessor(**self.sp_model_kwargs			)
								self.sp_model.Load(_SCREAMING_SNAKE_CASE			)

								# Used for whitespace normalization in input texts
								# fmt : off
								_UpperCAmelCase      =			{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
								# fmt : on

								# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
								_UpperCAmelCase      =			re.compile(
								    F"[{''.join(map(_SCREAMING_SNAKE_CASE						,		list(range(0						,		9			)			) + list(range(11						,		32			)			) + list(range(127						,		160			)			) + [160, 173, 8203]			)			)}]"			)






		def __getstate__(    self			):
								_UpperCAmelCase      =			self.__dict__.copy()
								_UpperCAmelCase      =			None
								return state






		def __setstate__(    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			d

								# for backward compatibility
								if not hasattr(self						,		"""sp_model_kwargs"""			):
														_UpperCAmelCase      =			{}

								_UpperCAmelCase      =			spm.SentencePieceProcessor(**self.sp_model_kwargs			)
								self.sp_model.Load(self.vocab_file			)






		@property
		# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
		def 				UpperCAmelCase (    self			):
								return len(self.sp_model			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			self.non_printing_characters_re.sub(""""""						,		_SCREAMING_SNAKE_CASE			)

								# Normalize whitespaces
								_UpperCAmelCase      =			"""""".join([char if char not in self.whitespaces else """ """ for char in text]			)

								# NFC Unicode normalization
								_UpperCAmelCase      =			unicodedata.normalize("""NFC"""						,		_SCREAMING_SNAKE_CASE			)
								return text






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		**_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			self.preprocess_text(_SCREAMING_SNAKE_CASE			)
								return self.sp_model.encode(_SCREAMING_SNAKE_CASE						,		out_type=_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								return self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								return self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE			)






		@staticmethod
		def 				UpperCAmelCase (    _SCREAMING_SNAKE_CASE			):
								return out_string






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			[]
								_UpperCAmelCase      =			""""""
								_UpperCAmelCase      =			False
								for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																				# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																				if not prev_is_special:
																										out_string += " "

																				out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE			) + token
																				_UpperCAmelCase      =			True
																				_UpperCAmelCase      =			[]
														else:
																				current_sub_tokens.append(_SCREAMING_SNAKE_CASE			)
																				_UpperCAmelCase      =			False
								out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE			)

								return out_string






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			{self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE			): i for i in range(self.vocab_size			)}
								vocab.update(self.added_tokens_encoder			)
								return vocab






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE = None			):
								if not os.path.isdir(_SCREAMING_SNAKE_CASE			):
														logger.error(F"Vocabulary path ({save_directory}) should be a directory"			)
														return
								_UpperCAmelCase      =			os.path.join(
								    _SCREAMING_SNAKE_CASE						,		(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

								if os.path.abspath(self.vocab_file			) != os.path.abspath(_SCREAMING_SNAKE_CASE			) and os.path.isfile(self.vocab_file			):
														copyfile(self.vocab_file						,		_SCREAMING_SNAKE_CASE			)
								elif not os.path.isfile(self.vocab_file			):
														with open(_SCREAMING_SNAKE_CASE						,		"""wb"""			) as fi:
																				_UpperCAmelCase      =			self.sp_model.serialized_model_proto()
																				fi.write(_SCREAMING_SNAKE_CASE			)

								return (out_vocab_file,)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE = False			):

								if isinstance(_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			):
														_UpperCAmelCase      =			self.preprocess_text(_SCREAMING_SNAKE_CASE			)
														_UpperCAmelCase      =			self.sp_model.encode(_SCREAMING_SNAKE_CASE			)
								else:
														_UpperCAmelCase      =			[self.preprocess_text(_SCREAMING_SNAKE_CASE			) for t in text]
														_UpperCAmelCase      =			self.sp_model.encode(_SCREAMING_SNAKE_CASE			)

								if return_tensors is True or return_tensors == "pt":
														_UpperCAmelCase      =			torch.tensor(_SCREAMING_SNAKE_CASE			)

								return token_ids






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								return self.sp_model.decode(_SCREAMING_SNAKE_CASE			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE			):
								_UpperCAmelCase      =			[F"User: {text}" if is_user else F"Bot: {text}" for is_user, text in conversation.iter_texts()]
								_UpperCAmelCase      =			(
								    F"{self.eos_token}{self.bos_token}" + F"{self.bos_token}".join(_SCREAMING_SNAKE_CASE			) + F"{self.bos_token}Bot:"
								)
								return self.encode(text=_SCREAMING_SNAKE_CASE			)





import logging

from transformers.configuration_utils import PretrainedConfig


a													=			logging.getLogger(__name__)




class       _A				(		__lowercase     ):
		__a					    =       """masked_bert"""






		def __init__(    self						,		_SCREAMING_SNAKE_CASE=3_0522						,		_SCREAMING_SNAKE_CASE=768						,		_SCREAMING_SNAKE_CASE=12						,		_SCREAMING_SNAKE_CASE=12						,		_SCREAMING_SNAKE_CASE=3072						,		_SCREAMING_SNAKE_CASE="gelu"						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=512						,		_SCREAMING_SNAKE_CASE=2						,		_SCREAMING_SNAKE_CASE=0.02						,		_SCREAMING_SNAKE_CASE=1e-12						,		_SCREAMING_SNAKE_CASE=0						,		_SCREAMING_SNAKE_CASE="topK"						,		_SCREAMING_SNAKE_CASE="constant"						,		_SCREAMING_SNAKE_CASE=0.0						,		**_SCREAMING_SNAKE_CASE						,		):
								super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE						,		**_SCREAMING_SNAKE_CASE			)

								_UpperCAmelCase      =			vocab_size
								_UpperCAmelCase      =			hidden_size
								_UpperCAmelCase      =			num_hidden_layers
								_UpperCAmelCase      =			num_attention_heads
								_UpperCAmelCase      =			hidden_act
								_UpperCAmelCase      =			intermediate_size
								_UpperCAmelCase      =			hidden_dropout_prob
								_UpperCAmelCase      =			attention_probs_dropout_prob
								_UpperCAmelCase      =			max_position_embeddings
								_UpperCAmelCase      =			type_vocab_size
								_UpperCAmelCase      =			initializer_range
								_UpperCAmelCase      =			layer_norm_eps
								_UpperCAmelCase      =			pruning_method
								_UpperCAmelCase      =			mask_init
								_UpperCAmelCase      =			mask_scale