





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
					import torch


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


a__    :		Any      						=     logging.get_logger(__name__)
a__    :		Union[str, Any]      						=     {'vocab_file': 'spiece.model'}

a__    :		Tuple      						=     {
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

a__    :		List[str]      						=     {
    'AI-Sweden/gpt-sw3-126m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-350m': 2_0_4_8,
    'AI-Sweden/gpt-sw3-1.6b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-6.7b': 2_0_4_8,
    'AI-Sweden/gpt-sw3-20b': 2_0_4_8,
}






class  lowercase    (      __lowercase       ):






					"""simple docstring"""



					snake_case_						  =			VOCAB_FILES_NAMES
					snake_case_						  =			PRETRAINED_VOCAB_FILES_MAP
					snake_case_						  =			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					snake_case_						  =			['input_ids', 'attention_mask']





					def __init__(    self						:     int ,					a_						:     List[Any] ,					a_						:     int=False ,					a_						:     int=False ,					a_						:     List[str]=False ,					a_						:     Optional[Any]=None ,					a_						:     Tuple=None ,					a_						:     Optional[Any]=None ,					a_						:     Any=None ,					a_						:     Optional[Dict[str, Any]] = None ,					**a_						:     Dict ,					):

						"""simple docstring"""



						lowerCamelCase__										=			{} if sp_model_kwargs is None else sp_model_kwargs

						lowerCamelCase__										=			kwargs.get("""name_or_path"""							)
						if name_or_path is None:
							logger.warning(
							    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
							    """ you are testing the model, this can safely be ignored"""							)
							lowerCamelCase__										=			"None"

						# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
						lowerCamelCase__										=			"<|endoftext|>" if eos_token is None else eos_token
						lowerCamelCase__										=			"<unk>" if unk_token is None else unk_token
						if "gpt-sw3-7b" in name_or_path:
							lowerCamelCase__										=			unk_token if pad_token is None else pad_token
							lowerCamelCase__										=			eos_token if bos_token is None else bos_token
						else:
							lowerCamelCase__										=			"<pad>" if pad_token is None else pad_token
							lowerCamelCase__										=			"<s>" if bos_token is None else bos_token

						super().__init__(
						    do_lower_case=a_ ,					remove_space=a_ ,					keep_accents=a_ ,					bos_token=a_ ,					eos_token=a_ ,					unk_token=a_ ,					pad_token=a_ ,					sp_model_kwargs=self.sp_model_kwargs ,					**a_ ,					)

						lowerCamelCase__										=			do_lower_case
						lowerCamelCase__										=			remove_space
						lowerCamelCase__										=			keep_accents
						lowerCamelCase__										=			vocab_file

						lowerCamelCase__										=			spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(a_							)

						# Used for whitespace normalization in input texts
						# fmt : off
						lowerCamelCase__										=			{" ", " ", " ", " ", " ", "　", " ", " ", " ", " ", "￼", ""}
						# fmt : on

						# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
						lowerCamelCase__										=			re.compile(
						    F'''[{''.join(map(a_ ,					list(range(0 ,					9							)							) + list(range(11 ,					32							)							) + list(range(1_27 ,					1_60							)							) + [1_60, 1_73, 82_03]							)							)}]'''							)





					def __getstate__(    self						:     Tuple							):

						"""simple docstring"""



						lowerCamelCase__										=			self.__dict__.copy()
						lowerCamelCase__										=			None
						return state





					def __setstate__(    self						:     Union[str, Any] ,					a_						:     Optional[int]							):

						"""simple docstring"""



						lowerCamelCase__										=			d

						# for backward compatibility
						if not hasattr(self ,					"""sp_model_kwargs"""							):
							lowerCamelCase__										=			{}

						lowerCamelCase__										=			spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(self.vocab_file							)





					@property
					# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
					def       _UpperCamelCase       (    self						:     int							):

						"""simple docstring"""



						return len(self.sp_model							)





					def       _UpperCamelCase       (    self						:     Any ,					a_						:     str							):

						"""simple docstring"""



						lowerCamelCase__										=			self.non_printing_characters_re.sub("""""" ,					a_							)

						# Normalize whitespaces
						lowerCamelCase__										=			"".join([char if char not in self.whitespaces else """ """ for char in text]							)

						# NFC Unicode normalization
						lowerCamelCase__										=			unicodedata.normalize("""NFC""" ,					a_							)
						return text





					def       _UpperCamelCase       (    self						:     int ,					a_						:     str ,					**a_						:     str							):

						"""simple docstring"""



						lowerCamelCase__										=			self.preprocess_text(a_							)
						return self.sp_model.encode(a_ ,					out_type=a_							)





					def       _UpperCamelCase       (    self						:     int ,					a_						:     str							):

						"""simple docstring"""



						return self.sp_model.PieceToId(a_							)





					def       _UpperCamelCase       (    self						:     Any ,					a_						:     int							):

						"""simple docstring"""



						return self.sp_model.IdToPiece(a_							)





					@staticmethod
					def       _UpperCamelCase       (    a_						:     str							):

						"""simple docstring"""



						return out_string





					def       _UpperCamelCase       (    self						:     Optional[int] ,					a_						:     List[str]							):

						"""simple docstring"""



						lowerCamelCase__										=			[]
						lowerCamelCase__										=			""
						lowerCamelCase__										=			False
						for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
								# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
								if not prev_is_special:
									out_string += " "

								out_string += self.sp_model.decode(a_							) + token
								lowerCamelCase__										=			True
								lowerCamelCase__										=			[]
							else:
								current_sub_tokens.append(a_							)
								lowerCamelCase__										=			False
						out_string += self.sp_model.decode(a_							)

						return out_string





					def       _UpperCamelCase       (    self						:     Union[str, Any]							):

						"""simple docstring"""



						lowerCamelCase__										=			{self.convert_ids_to_tokens(a_							): i for i in range(self.vocab_size							)}
						vocab.update(self.added_tokens_encoder							)
						return vocab





					def       _UpperCamelCase       (    self						:     Any ,					a_						:     str ,					a_						:     Optional[str] = None							):

						"""simple docstring"""



						if not os.path.isdir(a_							):
							logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''							)
							return
						lowerCamelCase__										=			os.path.join(
						    a_ ,					(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]							)

						if os.path.abspath(self.vocab_file							) != os.path.abspath(a_							) and os.path.isfile(self.vocab_file							):
							copyfile(self.vocab_file ,					a_							)
						elif not os.path.isfile(self.vocab_file							):
							with open(a_ ,					"""wb"""							) as fi:
								lowerCamelCase__										=			self.sp_model.serialized_model_proto()
								fi.write(a_							)

						return (out_vocab_file,)





					def       _UpperCamelCase       (    self						:     List[str] ,					a_						:     Union[str, List[str]] ,					a_						:     Union[str, bool] = False							):

						"""simple docstring"""



						if isinstance(a_ ,					a_							):
							lowerCamelCase__										=			self.preprocess_text(a_							)
							lowerCamelCase__										=			self.sp_model.encode(a_							)
						else:
							lowerCamelCase__										=			[self.preprocess_text(a_							) for t in text]
							lowerCamelCase__										=			self.sp_model.encode(a_							)

						if return_tensors is True or return_tensors == "pt":
							lowerCamelCase__										=			torch.tensor(a_							)

						return token_ids





					def       _UpperCamelCase       (    self						:     Any ,					a_						:     Union[int, List[int]]							):

						"""simple docstring"""



						return self.sp_model.decode(a_							)





					def       _UpperCamelCase       (    self						:     Dict ,					a_						:     "Conversation"							):

						"""simple docstring"""



						lowerCamelCase__										=			[F'''User: {text}''' if is_user else F'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
						lowerCamelCase__										=			(
						    F'''{self.eos_token}{self.bos_token}''' + F'''{self.bos_token}'''.join(a_							) + F'''{self.bos_token}Bot:'''
						)
						return self.encode(text=a_							)


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


lowercase    :   List[str]						=			{
    'configuration_gpt_bigcode': ['GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP', 'GPTBigCodeConfig'],
}

try:
		if not is_torch_available():
				raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
		pass
else:
		lowercase    :   List[str]						=			[
		    'GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST',
		    'GPTBigCodeForSequenceClassification',
		    'GPTBigCodeForTokenClassification',
		    'GPTBigCodeForCausalLM',
		    'GPTBigCodeModel',
		    'GPTBigCodePreTrainedModel',
		]

if TYPE_CHECKING:
		from .configuration_gpt_bigcode import GPT_BIGCODE_PRETRAINED_CONFIG_ARCHIVE_MAP, GPTBigCodeConfig

		try:
				if not is_torch_available():
						raise OptionalDependencyNotAvailable()
		except OptionalDependencyNotAvailable:
				pass
		else:
				from .modeling_gpt_bigcode import (
				    GPT_BIGCODE_PRETRAINED_MODEL_ARCHIVE_LIST,
				    GPTBigCodeForCausalLM,
				    GPTBigCodeForSequenceClassification,
				    GPTBigCodeForTokenClassification,
				    GPTBigCodeModel,
				    GPTBigCodePreTrainedModel,
				)


else:
		import sys

		lowercase    :   Tuple						=			_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)