





'''simple docstring'''

import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
    import torch


if TYPE_CHECKING:
    from transformers.pipelines.conversational import Conversation


a   	=    logging.get_logger(__name__)
a   	=    {"vocab_file": "spiece.model"}

a   	=    {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

a   	=    {
    "AI-Sweden/gpt-sw3-126m": 2048,
    "AI-Sweden/gpt-sw3-350m": 2048,
    "AI-Sweden/gpt-sw3-1.6b": 2048,
    "AI-Sweden/gpt-sw3-6.7b": 2048,
    "AI-Sweden/gpt-sw3-20b": 2048,
}







class __a			( _snake_case      ):
   __UpperCamelCase :			str						   =						VOCAB_FILES_NAMES
   __UpperCamelCase :			str						   =						PRETRAINED_VOCAB_FILES_MAP
   __UpperCamelCase :			List[Any]						   =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
   __UpperCamelCase :			List[Any]						   =						['input_ids', 'attention_mask']





   def __init__(       self					:      Optional[Any]					,lowerCamelCase					:      Union[str, Any]					,lowerCamelCase					:      Optional[int]=False					,lowerCamelCase					:      List[str]=False					,lowerCamelCase					:      Any=False					,lowerCamelCase					:      List[str]=None					,lowerCamelCase					:      List[Any]=None					,lowerCamelCase					:      Dict=None					,lowerCamelCase					:      str=None					,lowerCamelCase					:      Optional[Dict[str, Any]] = None					,**lowerCamelCase					:      str					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							{} if sp_model_kwargs is None else sp_model_kwargs

      __SCREAMING_SNAKE_CASE   				=							kwargs.get("""name_or_path"""     )
      if name_or_path is None:
         logger.warning(
             """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
             """ you are testing the model, this can safely be ignored"""     )
         __SCREAMING_SNAKE_CASE   				=							"""None"""

      # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
      __SCREAMING_SNAKE_CASE   				=							"""<|endoftext|>""" if eos_token is None else eos_token
      __SCREAMING_SNAKE_CASE   				=							"""<unk>""" if unk_token is None else unk_token
      if "gpt-sw3-7b" in name_or_path:
         __SCREAMING_SNAKE_CASE   				=							unk_token if pad_token is None else pad_token
         __SCREAMING_SNAKE_CASE   				=							eos_token if bos_token is None else bos_token
      else:
         __SCREAMING_SNAKE_CASE   				=							"""<pad>""" if pad_token is None else pad_token
         __SCREAMING_SNAKE_CASE   				=							"""<s>""" if bos_token is None else bos_token

      super().__init__(
          do_lower_case=lowerCamelCase					,remove_space=lowerCamelCase					,keep_accents=lowerCamelCase					,bos_token=lowerCamelCase					,eos_token=lowerCamelCase					,unk_token=lowerCamelCase					,pad_token=lowerCamelCase					,sp_model_kwargs=self.sp_model_kwargs					,**lowerCamelCase					,)

      __SCREAMING_SNAKE_CASE   				=							do_lower_case
      __SCREAMING_SNAKE_CASE   				=							remove_space
      __SCREAMING_SNAKE_CASE   				=							keep_accents
      __SCREAMING_SNAKE_CASE   				=							vocab_file

      __SCREAMING_SNAKE_CASE   				=							spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(lowerCamelCase     )

      # Used for whitespace normalization in input texts
      # fmt : off
      __SCREAMING_SNAKE_CASE   				=							{""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
      # fmt : on

      # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
      __SCREAMING_SNAKE_CASE   				=							re.compile(
          f"""[{''.join(map(lowerCamelCase					,list(range(0					,9     )     ) + list(range(11					,32     )     ) + list(range(127					,160     )     ) + [160, 173, 8203]     )     )}]"""     )





   def __getstate__(       self					:      Any     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.__dict__.copy()
      __SCREAMING_SNAKE_CASE   				=							None
      return state





   def __setstate__(       self					:      Union[str, Any]					,lowerCamelCase					:      Optional[int]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							d

      # for backward compatibility
      if not hasattr(self					,"""sp_model_kwargs"""     ):
         __SCREAMING_SNAKE_CASE   				=							{}

      __SCREAMING_SNAKE_CASE   				=							spm.SentencePieceProcessor(**self.sp_model_kwargs     )
      self.sp_model.Load(self.vocab_file     )





   @property
   # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
   def   UpperCAmelCase__  (       self					:      int     ):

      '''simple docstring'''

      return len(self.sp_model     )





   def   UpperCAmelCase__  (       self					:      List[str]					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.non_printing_characters_re.sub(""""""					,lowerCamelCase     )

      # Normalize whitespaces
      __SCREAMING_SNAKE_CASE   				=							"""""".join([char if char not in self.whitespaces else """ """ for char in text]     )

      # NFC Unicode normalization
      __SCREAMING_SNAKE_CASE   				=							unicodedata.normalize("""NFC"""					,lowerCamelCase     )
      return text





   def   UpperCAmelCase__  (       self					:      List[Any]					,lowerCamelCase					:      str					,**lowerCamelCase					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							self.preprocess_text(lowerCamelCase     )
      return self.sp_model.encode(lowerCamelCase					,out_type=lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      str					,lowerCamelCase					:      str     ):

      '''simple docstring'''

      return self.sp_model.PieceToId(lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      List[Any]					,lowerCamelCase					:      int     ):

      '''simple docstring'''

      return self.sp_model.IdToPiece(lowerCamelCase     )





   @staticmethod
   def   UpperCAmelCase__  (       lowerCamelCase					:      str     ):

      '''simple docstring'''

      return out_string





   def   UpperCAmelCase__  (       self					:      int					,lowerCamelCase					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[]
      __SCREAMING_SNAKE_CASE   				=							""""""
      __SCREAMING_SNAKE_CASE   				=							False
      for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
            # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
            if not prev_is_special:
               out_string += " "

            out_string += self.sp_model.decode(lowerCamelCase     ) + token
            __SCREAMING_SNAKE_CASE   				=							True
            __SCREAMING_SNAKE_CASE   				=							[]
         else:
            current_sub_tokens.append(lowerCamelCase     )
            __SCREAMING_SNAKE_CASE   				=							False
      out_string += self.sp_model.decode(lowerCamelCase     )

      return out_string





   def   UpperCAmelCase__  (       self					:      List[str]     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							{self.convert_ids_to_tokens(lowerCamelCase     ): i for i in range(self.vocab_size     )}
      vocab.update(self.added_tokens_encoder     )
      return vocab





   def   UpperCAmelCase__  (       self					:      str					,lowerCamelCase					:      str					,lowerCamelCase					:      Optional[str] = None     ):

      '''simple docstring'''

      if not os.path.isdir(lowerCamelCase     ):
         logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""     )
         return
      __SCREAMING_SNAKE_CASE   				=							os.path.join(
          lowerCamelCase					,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]     )

      if os.path.abspath(self.vocab_file     ) != os.path.abspath(lowerCamelCase     ) and os.path.isfile(self.vocab_file     ):
         copyfile(self.vocab_file					,lowerCamelCase     )
      elif not os.path.isfile(self.vocab_file     ):
         with open(lowerCamelCase					,"""wb"""     ) as fi:
            __SCREAMING_SNAKE_CASE   				=							self.sp_model.serialized_model_proto()
            fi.write(lowerCamelCase     )

      return (out_vocab_file,)





   def   UpperCAmelCase__  (       self					:      str					,lowerCamelCase					:      Union[str, List[str]]					,lowerCamelCase					:      Union[str, bool] = False     ):

      '''simple docstring'''

      if isinstance(lowerCamelCase					,lowerCamelCase     ):
         __SCREAMING_SNAKE_CASE   				=							self.preprocess_text(lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							self.sp_model.encode(lowerCamelCase     )
      else:
         __SCREAMING_SNAKE_CASE   				=							[self.preprocess_text(lowerCamelCase     ) for t in text]
         __SCREAMING_SNAKE_CASE   				=							self.sp_model.encode(lowerCamelCase     )

      if return_tensors is True or return_tensors == "pt":
         __SCREAMING_SNAKE_CASE   				=							torch.tensor(lowerCamelCase     )

      return token_ids





   def   UpperCAmelCase__  (       self					:      Optional[Any]					,lowerCamelCase					:      Union[int, List[int]]     ):

      '''simple docstring'''

      return self.sp_model.decode(lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Optional[int]					,lowerCamelCase					:      "Conversation"     ):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							[f"""User: {text}""" if is_user else f"""Bot: {text}""" for is_user, text in conversation.iter_texts()]
      __SCREAMING_SNAKE_CASE   				=							(
          f"""{self.eos_token}{self.bos_token}""" + f"""{self.bos_token}""".join(lowerCamelCase     ) + f"""{self.bos_token}Bot:"""
      )
      return self.encode(text=lowerCamelCase     )












'''simple docstring'''

from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import rescale, resize, to_channel_dimension_format
from ...image_utils import (
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
    import PIL


a   	=    logging.get_logger(__name__)

def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase   )    ->       str:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE   				=							b.T
   __SCREAMING_SNAKE_CASE   				=							np.sum(np.square(__UpperCAmelCase   )				,						axis=1   )
   __SCREAMING_SNAKE_CASE   				=							np.sum(np.square(__UpperCAmelCase   )				,						axis=0   )
   __SCREAMING_SNAKE_CASE   				=							np.matmul(__UpperCAmelCase				,						__UpperCAmelCase   )
   __SCREAMING_SNAKE_CASE   				=							aa[:, None] - 2 * ab + ba[None, :]
   return d

def     __magic_name__							(				__UpperCAmelCase				,						__UpperCAmelCase   )    ->       List[Any]:

   '''simple docstring'''







   __SCREAMING_SNAKE_CASE   				=							x.reshape(-1				,						3   )
   __SCREAMING_SNAKE_CASE   				=							squared_euclidean_distance(__UpperCAmelCase				,						__UpperCAmelCase   )
   return np.argmin(__UpperCAmelCase				,						axis=1   )







class __a			( _snake_case      ):
   __UpperCamelCase :			Any						   =						['pixel_values']





   def __init__(       self					:      Any					,lowerCamelCase					:      Optional[Union[List[List[int]], np.ndarray]] = None					,lowerCamelCase					:      bool = True					,lowerCamelCase					:      Dict[str, int] = None					,lowerCamelCase					:      PILImageResampling = PILImageResampling.BILINEAR					,lowerCamelCase					:      bool = True					,lowerCamelCase					:      bool = True					,**lowerCamelCase					:      Optional[Any]					,):

      '''simple docstring'''

      super().__init__(**lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							size if size is not None else {"""height""": 256, """width""": 256}
      __SCREAMING_SNAKE_CASE   				=							get_size_dict(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							np.array(lowerCamelCase     ) if clusters is not None else None
      __SCREAMING_SNAKE_CASE   				=							do_resize
      __SCREAMING_SNAKE_CASE   				=							size
      __SCREAMING_SNAKE_CASE   				=							resample
      __SCREAMING_SNAKE_CASE   				=							do_normalize
      __SCREAMING_SNAKE_CASE   				=							do_color_quantize





   def   UpperCAmelCase__  (       self					:      Optional[int]					,lowerCamelCase					:      np.ndarray					,lowerCamelCase					:      Dict[str, int]					,lowerCamelCase					:      PILImageResampling = PILImageResampling.BILINEAR					,lowerCamelCase					:      Optional[Union[str, ChannelDimension]] = None					,**lowerCamelCase					:      int					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							get_size_dict(lowerCamelCase     )
      if "height" not in size or "width" not in size:
         raise ValueError(f"""Size dictionary must contain both height and width keys. Got {size.keys()}"""     )
      return resize(
          lowerCamelCase					,size=(size["""height"""], size["""width"""])					,resample=lowerCamelCase					,data_format=lowerCamelCase					,**lowerCamelCase     )





   def   UpperCAmelCase__  (       self					:      Optional[Any]					,lowerCamelCase					:      np.ndarray					,lowerCamelCase					:      Optional[Union[str, ChannelDimension]] = None					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							rescale(image=lowerCamelCase					,scale=1 / 127.5					,data_format=lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							image - 1
      return image





   def   UpperCAmelCase__  (       self					:      Dict					,lowerCamelCase					:      ImageInput					,lowerCamelCase					:      bool = None					,lowerCamelCase					:      Dict[str, int] = None					,lowerCamelCase					:      PILImageResampling = None					,lowerCamelCase					:      bool = None					,lowerCamelCase					:      Optional[bool] = None					,lowerCamelCase					:      Optional[Union[List[List[int]], np.ndarray]] = None					,lowerCamelCase					:      Optional[Union[str, TensorType]] = None					,lowerCamelCase					:      Optional[Union[str, ChannelDimension]] = ChannelDimension.FIRST					,**lowerCamelCase					:      Any					,):

      '''simple docstring'''

      __SCREAMING_SNAKE_CASE   				=							do_resize if do_resize is not None else self.do_resize
      __SCREAMING_SNAKE_CASE   				=							size if size is not None else self.size
      __SCREAMING_SNAKE_CASE   				=							get_size_dict(lowerCamelCase     )
      __SCREAMING_SNAKE_CASE   				=							resample if resample is not None else self.resample
      __SCREAMING_SNAKE_CASE   				=							do_normalize if do_normalize is not None else self.do_normalize
      __SCREAMING_SNAKE_CASE   				=							do_color_quantize if do_color_quantize is not None else self.do_color_quantize
      __SCREAMING_SNAKE_CASE   				=							clusters if clusters is not None else self.clusters
      __SCREAMING_SNAKE_CASE   				=							np.array(lowerCamelCase     )

      __SCREAMING_SNAKE_CASE   				=							make_list_of_images(lowerCamelCase     )

      if not valid_images(lowerCamelCase     ):
         raise ValueError(
             """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
             """torch.Tensor, tf.Tensor or jax.ndarray."""     )

      if do_resize and size is None or resample is None:
         raise ValueError("""Size and resample must be specified if do_resize is True."""     )

      if do_color_quantize and clusters is None:
         raise ValueError("""Clusters must be specified if do_color_quantize is True."""     )

      # All transformations expect numpy arrays.
      __SCREAMING_SNAKE_CASE   				=							[to_numpy_array(lowerCamelCase     ) for image in images]

      if do_resize:
         __SCREAMING_SNAKE_CASE   				=							[self.resize(image=lowerCamelCase					,size=lowerCamelCase					,resample=lowerCamelCase     ) for image in images]

      if do_normalize:
         __SCREAMING_SNAKE_CASE   				=							[self.normalize(image=lowerCamelCase     ) for image in images]

      if do_color_quantize:
         __SCREAMING_SNAKE_CASE   				=							[to_channel_dimension_format(lowerCamelCase					,ChannelDimension.LAST     ) for image in images]
         # color quantize from (batch_size, height, width, 3) to (batch_size, height, width)
         __SCREAMING_SNAKE_CASE   				=							np.array(lowerCamelCase     )
         __SCREAMING_SNAKE_CASE   				=							color_quantize(lowerCamelCase					,lowerCamelCase     ).reshape(images.shape[:-1]     )

         # flatten to (batch_size, height*width)
         __SCREAMING_SNAKE_CASE   				=							images.shape[0]
         __SCREAMING_SNAKE_CASE   				=							images.reshape(lowerCamelCase					,-1     )

         # We need to convert back to a list of images to keep consistent behaviour across processors.
         __SCREAMING_SNAKE_CASE   				=							list(lowerCamelCase     )
      else:
         __SCREAMING_SNAKE_CASE   				=							[to_channel_dimension_format(lowerCamelCase					,lowerCamelCase     ) for image in images]

      __SCREAMING_SNAKE_CASE   				=							{"""input_ids""": images}
      return BatchFeature(data=lowerCamelCase					,tensor_type=lowerCamelCase     )





