





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
			import torch


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation


_a     :    List[Any]         =						logging.get_logger(__name__)
_a     :    Any         =						{'vocab_file': 'spiece.model'}

_a     :    List[Any]         =						{
    'vocab_file': {
        'AI-Sweden/gpt-sw3-126m': 'https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-350m': 'https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-1.6b': 'https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-6.7b': 'https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model',
        'AI-Sweden/gpt-sw3-20b': 'https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model',
    }
}

_a     :    List[Any]         =						{
    'AI-Sweden/gpt-sw3-126m': 2_048,
    'AI-Sweden/gpt-sw3-350m': 2_048,
    'AI-Sweden/gpt-sw3-1.6b': 2_048,
    'AI-Sweden/gpt-sw3-6.7b': 2_048,
    'AI-Sweden/gpt-sw3-20b': 2_048,
}
class   a_ (			_UpperCAmelCase      ):
			A__      :			List[str]       	=  VOCAB_FILES_NAMES
			A__      :			List[Any]       	=  PRETRAINED_VOCAB_FILES_MAP
			A__      :			Dict       	=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A__      :			int       	=  ['input_ids', 'attention_mask']

			def __init__(   self   :  Any     , UpperCAmelCase__   :  Dict     , UpperCAmelCase__   :  Union[str, Any]=False     , UpperCAmelCase__   :  List[Any]=False     , UpperCAmelCase__   :  int=False     , UpperCAmelCase__   :  List[str]=None     , UpperCAmelCase__   :  List[str]=None     , UpperCAmelCase__   :  Tuple=None     , UpperCAmelCase__   :  List[Any]=None     , UpperCAmelCase__   :  str = None     , **UpperCAmelCase__   :  Dict     , ):






							"""simple docstring"""
							snake_case   :					Optional[int]    						=				{} if sp_model_kwargs is None else sp_model_kwargs

							snake_case   :					Union[str, Any]    						=				kwargs.get('''name_or_path'''						)
							if name_or_path is None:
											logger.warning(
											    '''name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'''
											    ''' you are testing the model, this can safely be ignored'''						)
											snake_case   :					Dict    						=				'''None'''

							# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
							snake_case   :					Optional[int]    						=				'''<|endoftext|>''' if eos_token is None else eos_token
							snake_case   :					int    						=				'''<unk>''' if unk_token is None else unk_token
							if "gpt-sw3-7b" in name_or_path:
											snake_case   :					Optional[Any]    						=				unk_token if pad_token is None else pad_token
											snake_case   :					str    						=				eos_token if bos_token is None else bos_token
							else:
											snake_case   :					Tuple    						=				'''<pad>''' if pad_token is None else pad_token
											snake_case   :					Tuple    						=				'''<s>''' if bos_token is None else bos_token

							super().__init__(
							    do_lower_case=UpperCAmelCase__     , remove_space=UpperCAmelCase__     , keep_accents=UpperCAmelCase__     , bos_token=UpperCAmelCase__     , eos_token=UpperCAmelCase__     , unk_token=UpperCAmelCase__     , pad_token=UpperCAmelCase__     , sp_model_kwargs=self.sp_model_kwargs     , **UpperCAmelCase__     , )

							snake_case   :					Optional[Any]    						=				do_lower_case
							snake_case   :					Tuple    						=				remove_space
							snake_case   :					List[str]    						=				keep_accents
							snake_case   :					Union[str, Any]    						=				vocab_file

							snake_case   :					Tuple    						=				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(UpperCAmelCase__						)

							# Used for whitespace normalization in input texts
							# fmt : off
							snake_case   :					Dict    						=				{''' ''', ''' ''', ''' ''', ''' ''', ''' ''', '''　''', ''' ''', ''' ''', ''' ''', ''' ''', '''￼''', ''''''}
							# fmt : on

							# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
							snake_case   :					List[Any]    						=				re.compile(
							    F"[{''.join(map(UpperCAmelCase__     , list(range(0     , 9						)						) + list(range(11     , 32						)						) + list(range(127     , 160						)						) + [160, 173, 8_203]						)						)}]"						)

			def __getstate__(   self   :  Tuple						):






							"""simple docstring"""
							snake_case   :					List[str]    						=				self.__dict__.copy()
							snake_case   :					Dict    						=				None
							return state

			def __setstate__(   self   :  Optional[Any]     , UpperCAmelCase__   :  Dict						):






							"""simple docstring"""
							snake_case   :					Dict    						=				d

							# for backward compatibility
							if not hasattr(self     , '''sp_model_kwargs'''						):
											snake_case   :					Dict    						=				{}

							snake_case   :					Optional[Any]    						=				spm.SentencePieceProcessor(**self.sp_model_kwargs						)
							self.sp_model.Load(self.vocab_file						)

			@property
			# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
			def      lowerCAmelCase(   self   :  int						):






							"""simple docstring"""
							return len(self.sp_model						)

			def      lowerCAmelCase(   self   :  Union[str, Any]     , UpperCAmelCase__   :  List[str]						):






							"""simple docstring"""
							snake_case   :					int    						=				self.non_printing_characters_re.sub(''''''     , UpperCAmelCase__						)

							# Normalize whitespaces
							snake_case   :					Tuple    						=				''''''.join([char if char not in self.whitespaces else ''' ''' for char in text]						)

							# NFC Unicode normalization
							snake_case   :					str    						=				unicodedata.normalize('''NFC'''     , UpperCAmelCase__						)
							return text

			def      lowerCAmelCase(   self   :  List[Any]     , UpperCAmelCase__   :  List[str]     , **UpperCAmelCase__   :  Tuple						):






							"""simple docstring"""
							snake_case   :					Tuple    						=				self.preprocess_text(UpperCAmelCase__						)
							return self.sp_model.encode(UpperCAmelCase__     , out_type=UpperCAmelCase__						)

			def      lowerCAmelCase(   self   :  Dict     , UpperCAmelCase__   :  List[str]						):






							"""simple docstring"""
							return self.sp_model.PieceToId(UpperCAmelCase__						)

			def      lowerCAmelCase(   self   :  str     , UpperCAmelCase__   :  List[str]						):






							"""simple docstring"""
							return self.sp_model.IdToPiece(UpperCAmelCase__						)

			@staticmethod
			def      lowerCAmelCase(   UpperCAmelCase__   :  Any						):






							"""simple docstring"""
							return out_string

			def      lowerCAmelCase(   self   :  Union[str, Any]     , UpperCAmelCase__   :  List[str]						):






							"""simple docstring"""
							snake_case   :					Tuple    						=				[]
							snake_case   :					Tuple    						=				''''''
							snake_case   :					Dict    						=				False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
															if not prev_is_special:
																			out_string += " "

															out_string += self.sp_model.decode(UpperCAmelCase__						) + token
															snake_case   :					List[str]    						=				True
															snake_case   :					str    						=				[]
											else:
															current_sub_tokens.append(UpperCAmelCase__						)
															snake_case   :					Dict    						=				False
							out_string += self.sp_model.decode(UpperCAmelCase__						)

							return out_string

			def      lowerCAmelCase(   self   :  int						):






							"""simple docstring"""
							snake_case   :					Optional[int]    						=				{self.convert_ids_to_tokens(UpperCAmelCase__						): i for i in range(self.vocab_size						)}
							vocab.update(self.added_tokens_encoder						)
							return vocab

			def      lowerCAmelCase(   self   :  List[str]     , UpperCAmelCase__   :  Tuple     , UpperCAmelCase__   :  List[str] = None						):






							"""simple docstring"""
							if not os.path.isdir(UpperCAmelCase__						):
											logger.error(F"Vocabulary path ({save_directory}) should be a directory"						)
											return
							snake_case   :					List[str]    						=				os.path.join(
							    UpperCAmelCase__     , (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(UpperCAmelCase__						) and os.path.isfile(self.vocab_file						):
											copyfile(self.vocab_file     , UpperCAmelCase__						)
							elif not os.path.isfile(self.vocab_file						):
											with open(UpperCAmelCase__     , '''wb'''						) as fi:
															snake_case   :					Tuple    						=				self.sp_model.serialized_model_proto()
															fi.write(UpperCAmelCase__						)

							return (out_vocab_file,)

			def      lowerCAmelCase(   self   :  str     , UpperCAmelCase__   :  Union[str, Any]     , UpperCAmelCase__   :  Union[str, Any] = False						):






							"""simple docstring"""

							if isinstance(UpperCAmelCase__     , UpperCAmelCase__						):
											snake_case   :					List[str]    						=				self.preprocess_text(UpperCAmelCase__						)
											snake_case   :					Optional[int]    						=				self.sp_model.encode(UpperCAmelCase__						)
							else:
											snake_case   :					Union[str, Any]    						=				[self.preprocess_text(UpperCAmelCase__						) for t in text]
											snake_case   :					str    						=				self.sp_model.encode(UpperCAmelCase__						)

							if return_tensors is True or return_tensors == "pt":
											snake_case   :					List[Any]    						=				torch.tensor(UpperCAmelCase__						)

							return token_ids

			def      lowerCAmelCase(   self   :  List[str]     , UpperCAmelCase__   :  int						):






							"""simple docstring"""
							return self.sp_model.decode(UpperCAmelCase__						)



			def      lowerCAmelCase(   self   :  Union[str, Any]     , UpperCAmelCase__   :  Tuple						):






							"""simple docstring"""
							snake_case   :					Optional[Any]    						=				[F"User: {text}" if is_user else F"Bot: {text}" for is_user, text in conversation.iter_texts()]
							snake_case   :					Any    						=				(
							    F"{self.eos_token}{self.bos_token}" + F"{self.bos_token}".join(UpperCAmelCase__						) + F"{self.bos_token}Bot:"
							)
							return self.encode(text=UpperCAmelCase__						)







"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


SCREAMING_SNAKE_CASE__   =  logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__   =  {
    "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/config.json",
    "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/config.json"
    # See all FNet models at https://huggingface.co/models?filter=fnet
}

class 						lowercase   (   _UpperCAmelCase   ):
      _SCREAMING_SNAKE_CASE							      = 'fnet'





      def __init__(			self ,		lowercase=32_000 ,		lowercase=768 ,		lowercase=12 ,		lowercase=3_072 ,		lowercase="gelu_new" ,		lowercase=0.1 ,		lowercase=512 ,		lowercase=4 ,		lowercase=0.02 ,		lowercase=1e-12 ,		lowercase=False ,		lowercase=512 ,		lowercase=3 ,		lowercase=1 ,		lowercase=2 ,		**lowercase ,		)						->	int:
            super().__init__(pad_token_id=lowercase ,		bos_token_id=lowercase ,		eos_token_id=lowercase ,		**lowercase					)

            lowerCAmelCase    		=			vocab_size
            lowerCAmelCase    		=			max_position_embeddings
            lowerCAmelCase    		=			hidden_size
            lowerCAmelCase    		=			num_hidden_layers
            lowerCAmelCase    		=			intermediate_size
            lowerCAmelCase    		=			hidden_act
            lowerCAmelCase    		=			hidden_dropout_prob
            lowerCAmelCase    		=			initializer_range
            lowerCAmelCase    		=			type_vocab_size
            lowerCAmelCase    		=			layer_norm_eps
            lowerCAmelCase    		=			use_tpu_fourier_optimizations
            lowerCAmelCase    		=			tpu_short_seq_length






