




import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
							import torch


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


UpperCAmelCase_		:					Optional[int]          =  logging.get_logger(__name__)
UpperCAmelCase_		:					List[Any]          =  {"vocab_file": "spiece.model"}

UpperCAmelCase_		:					List[Any]          =  {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

UpperCAmelCase_		:					Optional[Any]          =  {
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}




class 		__A   (      UpperCamelCase__      ):
						UpperCamelCase      =       VOCAB_FILES_NAMES
						UpperCamelCase      =       PRETRAINED_VOCAB_FILES_MAP
						UpperCamelCase      =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCamelCase      =       ["""input_ids""", """attention_mask"""]

						def __init__(		self :List[Any]     ,			__snake_case :Union[str, Any]     ,			__snake_case :str=False     ,			__snake_case :List[str]=False     ,			__snake_case :str=False     ,			__snake_case :str=None     ,			__snake_case :List[str]=None     ,			__snake_case :Tuple=None     ,			__snake_case :Dict=None     ,			__snake_case :Optional[Dict[str, Any]] = None     ,			**__snake_case :Optional[Any]     ,			):
										'''simple docstring'''

										__magic_name__						:						Union[str, Any]										={} if sp_model_kwargs is None else sp_model_kwargs

										__magic_name__						:						List[Any]										=kwargs.get("""name_or_path"""			)
										if name_or_path is None:
														logger.warning(
														    """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
														    """ you are testing the model, this can safely be ignored"""			)
														__magic_name__						:						Any										="""None"""

										# Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
										__magic_name__						:						Optional[int]										="""<|endoftext|>""" if eos_token is None else eos_token
										__magic_name__						:						List[Any]										="""<unk>""" if unk_token is None else unk_token
										if "gpt-sw3-7b" in name_or_path:
														__magic_name__						:						int										=unk_token if pad_token is None else pad_token
														__magic_name__						:						int										=eos_token if bos_token is None else bos_token
										else:
														__magic_name__						:						Optional[Any]										="""<pad>""" if pad_token is None else pad_token
														__magic_name__						:						Optional[int]										="""<s>""" if bos_token is None else bos_token

										super().__init__(
										    do_lower_case=__snake_case     ,			remove_space=__snake_case     ,			keep_accents=__snake_case     ,			bos_token=__snake_case     ,			eos_token=__snake_case     ,			unk_token=__snake_case     ,			pad_token=__snake_case     ,			sp_model_kwargs=self.sp_model_kwargs     ,			**__snake_case     ,			)

										__magic_name__						:						Tuple										=do_lower_case
										__magic_name__						:						List[str]										=remove_space
										__magic_name__						:						Optional[Any]										=keep_accents
										__magic_name__						:						int										=vocab_file

										__magic_name__						:						str										=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.Load(__snake_case			)

										# Used for whitespace normalization in input texts
										# fmt : off
										__magic_name__						:						List[Any]										={""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
										# fmt : on

										# Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
										__magic_name__						:						str										=re.compile(
										    f"[{''.join(map(__snake_case     ,			list(range(0     ,			9			)			) + list(range(11     ,			32			)			) + list(range(1_27     ,			1_60			)			) + [1_60, 1_73, 82_03]			)			)}]"			)

						def __getstate__(		self :Dict			):
										'''simple docstring'''

										__magic_name__						:						Optional[int]										=self.__dict__.copy()
										__magic_name__						:						List[str]										=None
										return state

						def __setstate__(		self :List[Any]     ,			__snake_case :List[str]			):
										'''simple docstring'''

										__magic_name__						:						List[Any]										=d

										# for backward compatibility
										if not hasattr(self     ,			"""sp_model_kwargs"""			):
														__magic_name__						:						Dict										={}

										__magic_name__						:						Optional[int]										=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.Load(self.vocab_file			)

						@property
						# Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
						def      A__    (		self :Any			):
										'''simple docstring'''

										return len(self.sp_model			)

						def      A__    (		self :Any     ,			__snake_case :str			):
										'''simple docstring'''

										__magic_name__						:						Optional[int]										=self.non_printing_characters_re.sub(""""""     ,			__snake_case			)

										# Normalize whitespaces
										__magic_name__						:						Union[str, Any]										="""""".join([char if char not in self.whitespaces else """ """ for char in text]			)

										# NFC Unicode normalization
										__magic_name__						:						List[str]										=unicodedata.normalize("""NFC"""     ,			__snake_case			)
										return text

						def      A__    (		self :Dict     ,			__snake_case :str     ,			**__snake_case :Optional[int]			):
										'''simple docstring'''

										__magic_name__						:						str										=self.preprocess_text(__snake_case			)
										return self.sp_model.encode(__snake_case     ,			out_type=__snake_case			)

						def      A__    (		self :Union[str, Any]     ,			__snake_case :str			):
										'''simple docstring'''

										return self.sp_model.PieceToId(__snake_case			)

						def      A__    (		self :Dict     ,			__snake_case :int			):
										'''simple docstring'''

										return self.sp_model.IdToPiece(__snake_case			)

						@staticmethod
						def      A__    (		__snake_case :str			):
										'''simple docstring'''

										return out_string

						def      A__    (		self :Optional[int]     ,			__snake_case :List[str]			):
										'''simple docstring'''

										__magic_name__						:						Optional[int]										=[]
										__magic_name__						:						List[Any]										=""""""
										__magic_name__						:						Union[str, Any]										=False
										for token in tokens:
														# make sure that special tokens are not decoded using sentencepiece model
														if token in self.all_special_tokens:
																		# TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
																		if not prev_is_special:
																						out_string += " "

																		out_string += self.sp_model.decode(__snake_case			) + token
																		__magic_name__						:						int										=True
																		__magic_name__						:						str										=[]
														else:
																		current_sub_tokens.append(__snake_case			)
																		__magic_name__						:						List[Any]										=False
										out_string += self.sp_model.decode(__snake_case			)

										return out_string

						def      A__    (		self :Any			):
										'''simple docstring'''

										__magic_name__						:						List[Any]										={self.convert_ids_to_tokens(__snake_case			): i for i in range(self.vocab_size			)}
										vocab.update(self.added_tokens_encoder			)
										return vocab

						def      A__    (		self :List[str]     ,			__snake_case :str     ,			__snake_case :Optional[str] = None			):
										'''simple docstring'''

										if not os.path.isdir(__snake_case			):
														logger.error(f"Vocabulary path ({save_directory}) should be a directory"			)
														return
										__magic_name__						:						Tuple										=os.path.join(
										    __snake_case     ,			(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]			)

										if os.path.abspath(self.vocab_file			) != os.path.abspath(__snake_case			) and os.path.isfile(self.vocab_file			):
														copyfile(self.vocab_file     ,			__snake_case			)
										elif not os.path.isfile(self.vocab_file			):
														with open(__snake_case     ,			"""wb"""			) as fi:
																		__magic_name__						:						Dict										=self.sp_model.serialized_model_proto()
																		fi.write(__snake_case			)

										return (out_vocab_file,)

						def      A__    (		self :Dict     ,			__snake_case :Union[str, List[str]]     ,			__snake_case :Union[str, bool] = False			):
										'''simple docstring'''

										if isinstance(__snake_case     ,			__snake_case			):
														__magic_name__						:						List[str]										=self.preprocess_text(__snake_case			)
														__magic_name__						:						Optional[int]										=self.sp_model.encode(__snake_case			)
										else:
														__magic_name__						:						str										=[self.preprocess_text(__snake_case			) for t in text]
														__magic_name__						:						Optional[Any]										=self.sp_model.encode(__snake_case			)

										if return_tensors is True or return_tensors == "pt":
														__magic_name__						:						Any										=torch.tensor(__snake_case			)

										return token_ids

						def      A__    (		self :Tuple     ,			__snake_case :Union[int, List[int]]			):
										'''simple docstring'''

										return self.sp_model.decode(__snake_case			)


						def      A__    (		self :Any     ,			__snake_case :"Conversation"			):
										'''simple docstring'''

										__magic_name__						:						List[str]										=[f"User: {text}" if is_user else f"Bot: {text}" for is_user, text in conversation.iter_texts()]
										__magic_name__						:						str										=(
										    f"{self.eos_token}{self.bos_token}" + f"{self.bos_token}".join(__snake_case			) + f"{self.bos_token}Bot:"
										)
										return self.encode(text=__snake_case			)










from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
					import torch


__A								=       logging.get_logger(__name__)







class        __lowerCAmelCase					(			__magic_name__					):

			"""simple docstring"""







			snake_case_  						=			['''pixel_values''']


			def __init__( self     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = PILImageResampling.BILINEAR     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = 1 / 255     ,		lowerCamelCase__ = True     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> None:



							'''simple docstring'''
							super().__init__(**lowerCamelCase__				)
							__lowerCamelCase									=       size if size is not None else {'shortest_edge': 256}
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							__lowerCamelCase									=       crop_size if crop_size is not None else {'height': 224, 'width': 224}
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		param_name='crop_size'				)
							__lowerCamelCase									=       do_resize
							__lowerCamelCase									=       size
							__lowerCamelCase									=       resample
							__lowerCamelCase									=       do_center_crop
							__lowerCamelCase									=       crop_size
							__lowerCamelCase									=       do_rescale
							__lowerCamelCase									=       rescale_factor
							__lowerCamelCase									=       do_normalize
							__lowerCamelCase									=       image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
							__lowerCamelCase									=       image_std if image_std is not None else IMAGENET_STANDARD_STD


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = PILImageResampling.BICUBIC     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							if "shortest_edge" not in size:
											raise ValueError(f"""The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}"""				)
							__lowerCamelCase									=       get_resize_output_image_size(lowerCamelCase__     ,		size=size['shortest_edge']     ,		default_to_square=lowerCamelCase__				)
							return resize(lowerCamelCase__     ,		size=lowerCamelCase__     ,		resample=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__				)
							if "height" not in size or "width" not in size:
											raise ValueError(f"""The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}"""				)
							return center_crop(lowerCamelCase__     ,		size=(size['height'], size['width'])     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__				)       -> np.ndarray:



							'''simple docstring'''
							return rescale(lowerCamelCase__     ,		scale=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		**lowerCamelCase__     ,		)       -> np.ndarray:



							'''simple docstring'''
							return normalize(lowerCamelCase__     ,		mean=lowerCamelCase__     ,		std=lowerCamelCase__     ,		data_format=lowerCamelCase__     ,		**lowerCamelCase__				)


			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = None     ,		lowerCamelCase__ = ChannelDimension.FIRST     ,		**lowerCamelCase__     ,		)       -> str:



							'''simple docstring'''
							__lowerCamelCase									=       do_resize if do_resize is not None else self.do_resize
							__lowerCamelCase									=       size if size is not None else self.size
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		default_to_square=lowerCamelCase__				)
							__lowerCamelCase									=       resample if resample is not None else self.resample
							__lowerCamelCase									=       do_center_crop if do_center_crop is not None else self.do_center_crop
							__lowerCamelCase									=       crop_size if crop_size is not None else self.crop_size
							__lowerCamelCase									=       get_size_dict(lowerCamelCase__     ,		param_name='crop_size'				)
							__lowerCamelCase									=       do_rescale if do_rescale is not None else self.do_rescale
							__lowerCamelCase									=       rescale_factor if rescale_factor is not None else self.rescale_factor
							__lowerCamelCase									=       do_normalize if do_normalize is not None else self.do_normalize
							__lowerCamelCase									=       image_mean if image_mean is not None else self.image_mean
							__lowerCamelCase									=       image_std if image_std is not None else self.image_std

							__lowerCamelCase									=       make_list_of_images(lowerCamelCase__				)

							if not valid_images(lowerCamelCase__				):
											raise ValueError(
											    'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
											    'torch.Tensor, tf.Tensor or jax.ndarray.'				)

							if do_resize and size is None:
											raise ValueError('Size must be specified if do_resize is True.'				)

							if do_center_crop and crop_size is None:
											raise ValueError('Crop size must be specified if do_center_crop is True.'				)

							if do_rescale and rescale_factor is None:
											raise ValueError('Rescale factor must be specified if do_rescale is True.'				)

							if do_normalize and (image_mean is None or image_std is None):
											raise ValueError('Image mean and std must be specified if do_normalize is True.'				)

							# All transformations expect numpy arrays.
							__lowerCamelCase									=       [to_numpy_array(lowerCamelCase__				) for image in images]

							if do_resize:
											__lowerCamelCase									=       [self.resize(image=lowerCamelCase__     ,		size=lowerCamelCase__     ,		resample=lowerCamelCase__				) for image in images]

							if do_center_crop:
											__lowerCamelCase									=       [self.center_crop(image=lowerCamelCase__     ,		size=lowerCamelCase__				) for image in images]

							if do_rescale:
											__lowerCamelCase									=       [self.rescale(image=lowerCamelCase__     ,		scale=lowerCamelCase__				) for image in images]

							if do_normalize:
											__lowerCamelCase									=       [self.normalize(image=lowerCamelCase__     ,		mean=lowerCamelCase__     ,		std=lowerCamelCase__				) for image in images]

							__lowerCamelCase									=       [to_channel_dimension_format(lowerCamelCase__     ,		lowerCamelCase__				) for image in images]

							__lowerCamelCase									=       {'pixel_values': images}
							return BatchFeature(data=lowerCamelCase__     ,		tensor_type=lowerCamelCase__				)






			def  lowercase_       ( self     ,		lowerCamelCase__     ,		lowerCamelCase__ = None				)       -> Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase									=       outputs.logits

							# Resize logits and compute semantic segmentation maps
							if target_sizes is not None:
											if len(lowerCamelCase__				) != len(lowerCamelCase__				):
															raise ValueError(
															    'Make sure that you pass in as many target sizes as the batch dimension of the logits'				)

											if is_torch_tensor(lowerCamelCase__				):
															__lowerCamelCase									=       target_sizes.numpy()

											__lowerCamelCase									=       []

											for idx in range(len(lowerCamelCase__				)				):
															__lowerCamelCase									=       torch.nn.functional.interpolate(
															    logits[idx].unsqueeze(dim=0				)     ,		size=target_sizes[idx]     ,		mode='bilinear'     ,		align_corners=lowerCamelCase__				)
															__lowerCamelCase									=       resized_logits[0].argmax(dim=0				)
															semantic_segmentation.append(lowerCamelCase__				)
							else:
											__lowerCamelCase									=       logits.argmax(dim=1				)
											__lowerCamelCase									=       [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]				)]

							return semantic_segmentation


