




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
				import torch


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE__     :     List[Any]		    =       logging.get_logger(__name__)
SCREAMING_SNAKE_CASE__     :     int		    =       {"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE__     :     List[str]		    =       {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

SCREAMING_SNAKE_CASE__     :     Any		    =       {
    "AI-Sweden/gpt-sw3-126m": 20_48,
    "AI-Sweden/gpt-sw3-350m": 20_48,
    "AI-Sweden/gpt-sw3-1.6b": 20_48,
    "AI-Sweden/gpt-sw3-6.7b": 20_48,
    "AI-Sweden/gpt-sw3-20b": 20_48,
}



class       a_       (     SCREAMING_SNAKE_CASE__     ):
	A    							= VOCAB_FILES_NAMES
	A    							= PRETRAINED_VOCAB_FILES_MAP
	A    							= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	A    							= ['''input_ids''', '''attention_mask''']





	def __init__(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE=False      ,     SCREAMING_SNAKE_CASE=False      ,     SCREAMING_SNAKE_CASE=False      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE=None      ,     SCREAMING_SNAKE_CASE = None      ,     **SCREAMING_SNAKE_CASE      ,     )  ->					None:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						{} if sp_model_kwargs is None else sp_model_kwargs

				SCREAMING_SNAKE_CASE_								=						kwargs.get('name_or_path'      )
				if name_or_path is None:
							logger.warning(
							    'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
							    ' you are testing the model, this can safely be ignored'      )
							SCREAMING_SNAKE_CASE_								=						'None'

				# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
				SCREAMING_SNAKE_CASE_								=						'<|endoftext|>' if eos_token is None else eos_token
				SCREAMING_SNAKE_CASE_								=						'<unk>' if unk_token is None else unk_token
				if "gpt-sw3-7b" in name_or_path:
							SCREAMING_SNAKE_CASE_								=						unk_token if pad_token is None else pad_token
							SCREAMING_SNAKE_CASE_								=						eos_token if bos_token is None else bos_token
				else:
							SCREAMING_SNAKE_CASE_								=						'<pad>' if pad_token is None else pad_token
							SCREAMING_SNAKE_CASE_								=						'<s>' if bos_token is None else bos_token

				super().__init__(
				    do_lower_case=SCREAMING_SNAKE_CASE      ,     remove_space=SCREAMING_SNAKE_CASE      ,     keep_accents=SCREAMING_SNAKE_CASE      ,     bos_token=SCREAMING_SNAKE_CASE      ,     eos_token=SCREAMING_SNAKE_CASE      ,     unk_token=SCREAMING_SNAKE_CASE      ,     pad_token=SCREAMING_SNAKE_CASE      ,     sp_model_kwargs=self.sp_model_kwargs      ,     **SCREAMING_SNAKE_CASE      ,     )

				SCREAMING_SNAKE_CASE_								=						do_lower_case
				SCREAMING_SNAKE_CASE_								=						remove_space
				SCREAMING_SNAKE_CASE_								=						keep_accents
				SCREAMING_SNAKE_CASE_								=						vocab_file

				SCREAMING_SNAKE_CASE_								=						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
				self.sp_model.Load(SCREAMING_SNAKE_CASE      )

				# Used for whitespace normalization in input texts
				# fmt : off
				SCREAMING_SNAKE_CASE_								=						{' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
				# fmt : on

				# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
				SCREAMING_SNAKE_CASE_								=						re.compile(
				    f'[{"".join(map(SCREAMING_SNAKE_CASE      ,     list(range(0      ,     9      )      ) + list(range(11      ,     32      )      ) + list(range(127      ,     160      )      ) + [160, 173, 8203]      )      )}]'      )





	def __getstate__(							self      )  ->					Optional[int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self.__dict__.copy()
				SCREAMING_SNAKE_CASE_								=						None
				return state





	def __setstate__(							self      ,     SCREAMING_SNAKE_CASE      )  ->					Any:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						d

				# for backward compatibility
				if not hasattr(self      ,     'sp_model_kwargs'      ):
							SCREAMING_SNAKE_CASE_								=						{}

				SCREAMING_SNAKE_CASE_								=						spm.SentencePieceProcessor(**self.sp_model_kwargs      )
				self.sp_model.Load(self.vocab_file      )





	@property
	# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
	def 						A_(							self      )  ->					int:




				"""simple docstring"""



				return len(self.sp_model      )





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					str:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self.non_printing_characters_re.sub(''      ,     SCREAMING_SNAKE_CASE      )

				# Normalize whitespaces
				SCREAMING_SNAKE_CASE_								=						''.join([char if char not in self.whitespaces else ' ' for char in text]      )

				# NFC Unicode normalization
				SCREAMING_SNAKE_CASE_								=						unicodedata.normalize('NFC'      ,     SCREAMING_SNAKE_CASE      )
				return text





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     **SCREAMING_SNAKE_CASE      )  ->					List[str]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						self.preprocess_text(SCREAMING_SNAKE_CASE      )
				return self.sp_model.encode(SCREAMING_SNAKE_CASE      ,     out_type=SCREAMING_SNAKE_CASE      )





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					int:




				"""simple docstring"""



				return self.sp_model.PieceToId(SCREAMING_SNAKE_CASE      )





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					str:




				"""simple docstring"""



				return self.sp_model.IdToPiece(SCREAMING_SNAKE_CASE      )





	@staticmethod
	def 						A_(							SCREAMING_SNAKE_CASE      )  ->					str:




				"""simple docstring"""



				return out_string





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					str:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[]
				SCREAMING_SNAKE_CASE_								=						''
				SCREAMING_SNAKE_CASE_								=						False
				for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
										# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
										if not prev_is_special:
													out_string += " "

										out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE      ) + token
										SCREAMING_SNAKE_CASE_								=						True
										SCREAMING_SNAKE_CASE_								=						[]
							else:
										current_sub_tokens.append(SCREAMING_SNAKE_CASE      )
										SCREAMING_SNAKE_CASE_								=						False
				out_string += self.sp_model.decode(SCREAMING_SNAKE_CASE      )

				return out_string





	def 						A_(							self      )  ->					Dict[str, int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						{self.convert_ids_to_tokens(SCREAMING_SNAKE_CASE      ): i for i in range(self.vocab_size      )}
				vocab.update(self.added_tokens_encoder      )
				return vocab





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = None      )  ->					Tuple[str]:




				"""simple docstring"""



				if not os.path.isdir(SCREAMING_SNAKE_CASE      ):
							logger.error(f'Vocabulary path ({save_directory}) should be a directory'      )
							return
				SCREAMING_SNAKE_CASE_								=						os.path.join(
				    SCREAMING_SNAKE_CASE      ,     (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']      )

				if os.path.abspath(self.vocab_file      ) != os.path.abspath(SCREAMING_SNAKE_CASE      ) and os.path.isfile(self.vocab_file      ):
							copyfile(self.vocab_file      ,     SCREAMING_SNAKE_CASE      )
				elif not os.path.isfile(self.vocab_file      ):
							with open(SCREAMING_SNAKE_CASE      ,     'wb'      ) as fi:
										SCREAMING_SNAKE_CASE_								=						self.sp_model.serialized_model_proto()
										fi.write(SCREAMING_SNAKE_CASE      )

				return (out_vocab_file,)





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE = False      )  ->					Union[List[int], List[List[int]], "torch.Tensor"]:




				"""simple docstring"""



				if isinstance(SCREAMING_SNAKE_CASE      ,     SCREAMING_SNAKE_CASE      ):
							SCREAMING_SNAKE_CASE_								=						self.preprocess_text(SCREAMING_SNAKE_CASE      )
							SCREAMING_SNAKE_CASE_								=						self.sp_model.encode(SCREAMING_SNAKE_CASE      )
				else:
							SCREAMING_SNAKE_CASE_								=						[self.preprocess_text(SCREAMING_SNAKE_CASE      ) for t in text]
							SCREAMING_SNAKE_CASE_								=						self.sp_model.encode(SCREAMING_SNAKE_CASE      )

				if return_tensors is True or return_tensors == "pt":
							SCREAMING_SNAKE_CASE_								=						torch.tensor(SCREAMING_SNAKE_CASE      )

				return token_ids





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					str:




				"""simple docstring"""



				return self.sp_model.decode(SCREAMING_SNAKE_CASE      )





	def 						A_(							self      ,     SCREAMING_SNAKE_CASE      )  ->					List[int]:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						[f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
				SCREAMING_SNAKE_CASE_								=						(
				    f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(SCREAMING_SNAKE_CASE      ) + f'{self.bos_token}Bot:'
				)
				return self.encode(text=SCREAMING_SNAKE_CASE      )






from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow


if is_tf_available():
				import tensorflow as tf

				from transformers import AutoTokenizer, TFAutoModelForSeqaSeqLM



@require_tf
@require_sentencepiece
@require_tokenizers
class       a_       (     unittest.TestCase     ):





	@slow
	def 						A_(							self      )  ->					str:




				"""simple docstring"""



				SCREAMING_SNAKE_CASE_								=						TFAutoModelForSeqaSeqLM.from_pretrained('google/mt5-small'      )
				SCREAMING_SNAKE_CASE_								=						AutoTokenizer.from_pretrained('google/mt5-small'      )

				SCREAMING_SNAKE_CASE_								=						tokenizer('Hello there'      ,     return_tensors='tf'      ).input_ids
				SCREAMING_SNAKE_CASE_								=						tokenizer('Hi I am'      ,     return_tensors='tf'      ).input_ids

				SCREAMING_SNAKE_CASE_								=						model(SCREAMING_SNAKE_CASE      ,     labels=SCREAMING_SNAKE_CASE      ).loss
				SCREAMING_SNAKE_CASE_								=						-tf.math.reduce_mean(SCREAMING_SNAKE_CASE      ).numpy()

				SCREAMING_SNAKE_CASE_								=						-2_1.2_2_8_1_6_8
				self.assertTrue(abs(mtf_score - EXPECTED_SCORE      ) < 2e-4      )
