






"""simple docstring"""






import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
					import torch


if TYPE_CHECKING:
					from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE						:						str					   =       logging.get_logger(__name__)
SCREAMING_SNAKE_CASE						:						Optional[int]					   =       {'''vocab_file''': '''spiece.model'''}

SCREAMING_SNAKE_CASE						:						Tuple					   =       {
    '''vocab_file''': {
        '''AI-Sweden/gpt-sw3-126m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-350m''': '''https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-1.6b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-6.7b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model''',
        '''AI-Sweden/gpt-sw3-20b''': '''https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model''',
    }
}

SCREAMING_SNAKE_CASE						:						str					   =       {
    '''AI-Sweden/gpt-sw3-126m''': 2_048,
    '''AI-Sweden/gpt-sw3-350m''': 2_048,
    '''AI-Sweden/gpt-sw3-1.6b''': 2_048,
    '''AI-Sweden/gpt-sw3-6.7b''': 2_048,
    '''AI-Sweden/gpt-sw3-20b''': 2_048,
}





class       snake_case_       (    _lowerCamelCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							Tuple   			=					VOCAB_FILES_NAMES
	SCREAMING_SNAKE_CASE_:							Union[str, Any]   			=					PRETRAINED_VOCAB_FILES_MAP
	SCREAMING_SNAKE_CASE_:							Any   			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
	SCREAMING_SNAKE_CASE_:							Optional[Any]   			=					["""input_ids""", """attention_mask"""]


	def __init__(      self  ,					__a  ,					__a=False  ,					__a=False  ,					__a=False  ,					__a=None  ,					__a=None  ,					__a=None  ,					__a=None  ,					__a = None  ,					**__a  ,					):


					"""simple docstring"""

					A__							 =       {} if sp_model_kwargs is None else sp_model_kwargs

					A__							 =       kwargs.get('name_or_path'						)
					if name_or_path is None:
									logger.warning(
									    'name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,'
									    ' you are testing the model, this can safely be ignored'						)
									A__							 =       'None'

					# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
					A__							 =       '<|endoftext|>' if eos_token is None else eos_token
					A__							 =       '<unk>' if unk_token is None else unk_token
					if "gpt-sw3-7b" in name_or_path:
									A__							 =       unk_token if pad_token is None else pad_token
									A__							 =       eos_token if bos_token is None else bos_token
					else:
									A__							 =       '<pad>' if pad_token is None else pad_token
									A__							 =       '<s>' if bos_token is None else bos_token

					super().__init__(
					    do_lower_case=__a  ,					remove_space=__a  ,					keep_accents=__a  ,					bos_token=__a  ,					eos_token=__a  ,					unk_token=__a  ,					pad_token=__a  ,					sp_model_kwargs=self.sp_model_kwargs  ,					**__a  ,					)

					A__							 =       do_lower_case
					A__							 =       remove_space
					A__							 =       keep_accents
					A__							 =       vocab_file

					A__							 =       spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(__a						)

					# Used for whitespace normalization in input texts
					# fmt : off
					A__							 =       {' ', ' ', ' ', ' ', ' ', '　', ' ', ' ', ' ', ' ', '￼', ''}
					# fmt : on

					# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
					A__							 =       re.compile(
					    f'''[{''.join(map(__a  ,					list(range(0  ,					9						)						) + list(range(11  ,					32						)						) + list(range(127  ,					160						)						) + [160, 173, 8203]						)						)}]'''						)


	def __getstate__(      self						):


					"""simple docstring"""

					A__							 =       self.__dict__.copy()
					A__							 =       None
					return state


	def __setstate__(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       d

					# for backward compatibility
					if not hasattr(self  ,					'sp_model_kwargs'						):
									A__							 =       {}

					A__							 =       spm.SentencePieceProcessor(**self.sp_model_kwargs						)
					self.sp_model.Load(self.vocab_file						)


	@property
	# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					return len(self.sp_model						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       self.non_printing_characters_re.sub(''  ,					__a						)

					# Normalize whitespaces
					A__							 =       ''.join([char if char not in self.whitespaces else ' ' for char in text]						)

					# NFC Unicode normalization
					A__							 =       unicodedata.normalize('NFC'  ,					__a						)
					return text


	def       _UpperCAmelCase						(      self  ,					__a  ,					**__a						):


					"""simple docstring"""

					A__							 =       self.preprocess_text(__a						)
					return self.sp_model.encode(__a  ,					out_type=__a						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					return self.sp_model.PieceToId(__a						)


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					return self.sp_model.IdToPiece(__a						)


	@staticmethod
	def       _UpperCAmelCase						(      __a						):


					"""simple docstring"""

					return out_string


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       []
					A__							 =       ''
					A__							 =       False
					for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
													# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
													if not prev_is_special:
																	out_string += " "

													out_string += self.sp_model.decode(__a						) + token
													A__							 =       True
													A__							 =       []
									else:
													current_sub_tokens.append(__a						)
													A__							 =       False
					out_string += self.sp_model.decode(__a						)

					return out_string


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       {self.convert_ids_to_tokens(__a						): i for i in range(self.vocab_size						)}
					vocab.update(self.added_tokens_encoder						)
					return vocab


	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = None						):


					"""simple docstring"""

					if not os.path.isdir(__a						):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
									return
					A__							 =       os.path.join(
					    __a  ,					(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']						)

					if os.path.abspath(self.vocab_file						) != os.path.abspath(__a						) and os.path.isfile(self.vocab_file						):
									copyfile(self.vocab_file  ,					__a						)
					elif not os.path.isfile(self.vocab_file						):
									with open(__a  ,					'wb'						) as fi:
													A__							 =       self.sp_model.serialized_model_proto()
													fi.write(__a						)

					return (out_vocab_file,)


	def       _UpperCAmelCase						(      self  ,					__a  ,					__a = False						):


					"""simple docstring"""

					if isinstance(__a  ,					__a						):
									A__							 =       self.preprocess_text(__a						)
									A__							 =       self.sp_model.encode(__a						)
					else:
									A__							 =       [self.preprocess_text(__a						) for t in text]
									A__							 =       self.sp_model.encode(__a						)

					if return_tensors is True or return_tensors == "pt":
									A__							 =       torch.tensor(__a						)

					return token_ids


	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					return self.sp_model.decode(__a						)






	def       _UpperCAmelCase						(      self  ,					__a						):


					"""simple docstring"""

					A__							 =       [f'''User: {text}''' if is_user else f'''Bot: {text}''' for is_user, text in conversation.iter_texts()]
					A__							 =       (
					    f'''{self.eos_token}{self.bos_token}''' + f'''{self.bos_token}'''.join(__a						) + f'''{self.bos_token}Bot:'''
					)
					return self.encode(text=__a						)







"""simple docstring"""






import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin, prepare_image_inputs


if is_torch_available():
					import torch

if is_vision_available():
					from PIL import Image

					from transformers import PoolFormerImageProcessor





class       snake_case_       (    unittest.TestCase       ):



	"""simple docstring"""

	def __init__(      self  ,					__a  ,					__a=7  ,					__a=3  ,					__a=30  ,					__a=400  ,					__a=True  ,					__a=None  ,					__a=0.9  ,					__a=None  ,					__a=True  ,					__a=[0.5, 0.5, 0.5]  ,					__a=[0.5, 0.5, 0.5]  ,					):


					"""simple docstring"""

					A__							 =       size if size is not None else {'shortest_edge': 30}
					A__							 =       crop_size if crop_size is not None else {'height': 30, 'width': 30}
					A__							 =       parent
					A__							 =       batch_size
					A__							 =       num_channels
					A__							 =       min_resolution
					A__							 =       max_resolution
					A__							 =       do_resize_and_center_crop
					A__							 =       size
					A__							 =       crop_pct
					A__							 =       crop_size
					A__							 =       do_normalize
					A__							 =       image_mean
					A__							 =       image_std






	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					return {
					    "size": self.size,
					    "do_resize_and_center_crop": self.do_resize_and_center_crop,
					    "crop_pct": self.crop_pct,
					    "crop_size": self.crop_size,
					    "do_normalize": self.do_normalize,
					    "image_mean": self.image_mean,
					    "image_std": self.image_std,
					}





@require_torch
@require_vision
class       snake_case_       (    _lowerCamelCase    ,							unittest.TestCase       ):



	"""simple docstring"""

	SCREAMING_SNAKE_CASE_:							List[Any]   			=					PoolFormerImageProcessor if is_vision_available() else None


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       PoolFormerImageProcessingTester(self						)


	@property
	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					return self.image_processor_tester.prepare_image_processor_dict()


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.image_processing_class(**self.image_processor_dict						)
					self.assertTrue(hasattr(__a  ,					'do_resize_and_center_crop'						)						)
					self.assertTrue(hasattr(__a  ,					'size'						)						)
					self.assertTrue(hasattr(__a  ,					'crop_pct'						)						)
					self.assertTrue(hasattr(__a  ,					'do_normalize'						)						)
					self.assertTrue(hasattr(__a  ,					'image_mean'						)						)
					self.assertTrue(hasattr(__a  ,					'image_std'						)						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.image_processing_class.from_dict(self.image_processor_dict						)
					self.assertEqual(image_processor.size  ,					{'shortest_edge': 30}						)
					self.assertEqual(image_processor.crop_size  ,					{'height': 30, 'width': 30}						)

					A__							 =       self.image_processing_class.from_dict(self.image_processor_dict  ,					size=42  ,					crop_size=84						)
					self.assertEqual(image_processor.size  ,					{'shortest_edge': 42}						)
					self.assertEqual(image_processor.crop_size  ,					{'height': 84, 'width': 84}						)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					pass


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.image_processing_class(**self.image_processor_dict						)
					# create random PIL images
					A__							 =       prepare_image_inputs(self.image_processor_tester  ,					equal_resolution=__a						)
					for image in image_inputs:
									self.assertIsInstance(__a  ,					Image.Image						)

					# Test not batched input
					A__							 =       image_processing(image_inputs[0]  ,					return_tensors='pt'						).pixel_values

					self.assertEqual(
					    encoded_images.shape  ,					(
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )  ,					)

					# Test batched
					A__							 =       image_processing(__a  ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape  ,					(
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )  ,					)


	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.image_processing_class(**self.image_processor_dict						)
					# create random numpy tensors
					A__							 =       prepare_image_inputs(self.image_processor_tester  ,					equal_resolution=__a  ,					numpify=__a						)
					for image in image_inputs:
									self.assertIsInstance(__a  ,					np.ndarray						)

					# Test not batched input
					A__							 =       image_processing(image_inputs[0]  ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape  ,					(
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )  ,					)

					# Test batched
					A__							 =       image_processing(__a  ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape  ,					(
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )  ,					)






	def       _UpperCAmelCase						(      self						):


					"""simple docstring"""

					A__							 =       self.image_processing_class(**self.image_processor_dict						)
					# create random PyTorch tensors
					A__							 =       prepare_image_inputs(self.image_processor_tester  ,					equal_resolution=__a  ,					torchify=__a						)
					for image in image_inputs:
									self.assertIsInstance(__a  ,					torch.Tensor						)

					# Test not batched input
					A__							 =       image_processing(image_inputs[0]  ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape  ,					(
					        1,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )  ,					)

					# Test batched
					A__							 =       image_processing(__a  ,					return_tensors='pt'						).pixel_values
					self.assertEqual(
					    encoded_images.shape  ,					(
					        self.image_processor_tester.batch_size,
					        self.image_processor_tester.num_channels,
					        self.image_processor_tester.crop_size['height'],
					        self.image_processor_tester.crop_size['width'],
					    )  ,					)


