






"""simple docstring"""







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


_lowerCAmelCase            =	logging.get_logger(__name__)

_lowerCAmelCase            =	{'tokenizer_file': 'tokenizer.json'}

_lowerCAmelCase            =	{
    'tokenizer_file': {
        'bigscience/tokenizer': 'https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json',
        'bigscience/bloom-560m': 'https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json',
        'bigscience/bloom-1b1': 'https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json',
        'bigscience/bloom-1b7': 'https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json',
        'bigscience/bloom-3b': 'https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json',
        'bigscience/bloom-7b1': 'https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json',
        'bigscience/bloom': 'https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json',
    },
}


class      UpperCamelCase		(__snake_case     ):
						_SCREAMING_SNAKE_CASE  : str												=		VOCAB_FILES_NAMES
						_SCREAMING_SNAKE_CASE  : Optional[Any]												=		PRETRAINED_VOCAB_FILES_MAP
						_SCREAMING_SNAKE_CASE  : Tuple												=		["""input_ids""", """attention_mask"""]
						_SCREAMING_SNAKE_CASE  : Tuple												=		None






						def __init__(		self  :List[str]			, __magic_name__  :List[str]=None			, __magic_name__  :List[str]=None			, __magic_name__  :Dict=None			, __magic_name__  :Optional[int]="<unk>"			, __magic_name__  :List[str]="<s>"			, __magic_name__  :Tuple="</s>"			, __magic_name__  :Dict="<pad>"			, __magic_name__  :Optional[int]=False			, __magic_name__  :Any=False			, **__magic_name__  :List[str]			, )      ->Optional[Any]:
										super().__init__(
										    __magic_name__			, __magic_name__			, tokenizer_file=__magic_name__			, unk_token=__magic_name__			, bos_token=__magic_name__			, eos_token=__magic_name__			, pad_token=__magic_name__			, add_prefix_space=__magic_name__			, clean_up_tokenization_spaces=__magic_name__			, **__magic_name__			, )
										lowercase    :      List[str]       = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()				)
										if pre_tok_state.get("""add_prefix_space"""			, __magic_name__				) != add_prefix_space:
														lowercase    :      List[str]       = getattr(__magic_name__			, pre_tok_state.pop("""type"""				)				)
														lowercase    :      Any       = add_prefix_space
														lowercase    :      Union[str, Any]       = pre_tok_class(**__magic_name__				)

										lowercase    :      Optional[int]       = add_prefix_space






						def      __snake_case      (		self  :int			, *__magic_name__  :Any			, **__magic_name__  :List[str]				)      ->BatchEncoding:
										lowercase    :      Union[str, Any]       = kwargs.get("""is_split_into_words"""			, __magic_name__				)
										if not (self.add_prefix_space or not is_split_into_words):
														raise Exception(
														    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
														    """ pretokenized inputs."""				)

										return super()._batch_encode_plus(*__magic_name__			, **__magic_name__				)






						def      __snake_case      (		self  :int			, *__magic_name__  :Optional[Any]			, **__magic_name__  :Optional[int]				)      ->BatchEncoding:
										lowercase    :      List[Any]       = kwargs.get("""is_split_into_words"""			, __magic_name__				)

										if not (self.add_prefix_space or not is_split_into_words):
														raise Exception(
														    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with"""
														    """ pretokenized inputs."""				)

										return super()._encode_plus(*__magic_name__			, **__magic_name__				)






						def      __snake_case      (		self  :Dict			, __magic_name__  :str			, __magic_name__  :Optional[str] = None				)      ->Tuple[str]:
										lowercase    :      List[Any]       = self._tokenizer.model.save(__magic_name__			, name=__magic_name__				)
										return tuple(__magic_name__				)






						def      __snake_case      (		self  :str			, __magic_name__  :"Conversation"				)      ->List[int]:
										lowercase    :      str       = []
										for is_user, text in conversation.iter_texts():
														input_ids.extend(self.encode(__magic_name__			, add_special_tokens=__magic_name__				) + [self.eos_token_id]				)

										if len(__magic_name__				) > self.model_max_length:
														lowercase    :      List[Any]       = input_ids[-self.model_max_length :]
										return input_ids








"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
				import PIL

_lowerCAmelCase            =	logging.get_logger(__name__)


def      UpperCamelCase       ( _A )     ->       List[List[ImageInput]]:
				if isinstance(_A  ,				(list, tuple) ) and isinstance(videos[0]  ,				(list, tuple) ) and is_valid_image(videos[0][0] ):
								return videos

				elif isinstance(_A  ,				(list, tuple) ) and is_valid_image(videos[0] ):
								return [videos]

				elif is_valid_image(_A ):
								return [[videos]]

				raise ValueError(F"""Could not make batched video from {videos}""" )


class      UpperCamelCase		(__snake_case     ):
						_SCREAMING_SNAKE_CASE  : str												=		["""pixel_values"""]






						def __init__(		self  :str			, __magic_name__  :bool = True			, __magic_name__  :Dict[str, int] = None			, __magic_name__  :PILImageResampling = PILImageResampling.BILINEAR			, __magic_name__  :bool = True			, __magic_name__  :Dict[str, int] = None			, __magic_name__  :bool = True			, __magic_name__  :Union[int, float] = 1 / 255			, __magic_name__  :bool = True			, __magic_name__  :bool = True			, __magic_name__  :Optional[Union[float, List[float]]] = None			, __magic_name__  :Optional[Union[float, List[float]]] = None			, **__magic_name__  :List[str]			, )      ->None:
										super().__init__(**__magic_name__				)
										lowercase    :      Union[str, Any]       = size if size is not None else {"""shortest_edge""": 256}
										lowercase    :      Tuple       = get_size_dict(__magic_name__			, default_to_square=__magic_name__				)
										lowercase    :      List[Any]       = crop_size if crop_size is not None else {"""height""": 224, """width""": 224}
										lowercase    :      List[str]       = get_size_dict(__magic_name__			, param_name="""crop_size"""				)

										lowercase    :      Tuple       = do_resize
										lowercase    :      Optional[Any]       = size
										lowercase    :      Union[str, Any]       = do_center_crop
										lowercase    :      List[Any]       = crop_size
										lowercase    :      Union[str, Any]       = resample
										lowercase    :      Optional[Any]       = do_rescale
										lowercase    :      int       = rescale_factor
										lowercase    :      Dict       = offset
										lowercase    :      Optional[int]       = do_normalize
										lowercase    :      Optional[Any]       = image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
										lowercase    :      Tuple       = image_std if image_std is not None else IMAGENET_STANDARD_STD






						def      __snake_case      (		self  :Any			, __magic_name__  :np.ndarray			, __magic_name__  :Dict[str, int]			, __magic_name__  :PILImageResampling = PILImageResampling.BILINEAR			, __magic_name__  :Optional[Union[str, ChannelDimension]] = None			, **__magic_name__  :Optional[Any]			, )      ->np.ndarray:
										lowercase    :      Optional[int]       = get_size_dict(__magic_name__			, default_to_square=__magic_name__				)
										if "shortest_edge" in size:
														lowercase    :      int       = get_resize_output_image_size(__magic_name__			, size["""shortest_edge"""]			, default_to_square=__magic_name__				)
										elif "height" in size and "width" in size:
														lowercase    :      Tuple       = (size["""height"""], size["""width"""])
										else:
														raise ValueError(f"""Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"""				)
										return resize(__magic_name__			, size=__magic_name__			, resample=__magic_name__			, data_format=__magic_name__			, **__magic_name__				)






						def      __snake_case      (		self  :Optional[Any]			, __magic_name__  :np.ndarray			, __magic_name__  :Dict[str, int]			, __magic_name__  :Optional[Union[str, ChannelDimension]] = None			, **__magic_name__  :str			, )      ->np.ndarray:
										lowercase    :      Any       = get_size_dict(__magic_name__				)
										if "height" not in size or "width" not in size:
														raise ValueError(f"""Size must have 'height' and 'width' as keys. Got {size.keys()}"""				)
										return center_crop(__magic_name__			, size=(size["""height"""], size["""width"""])			, data_format=__magic_name__			, **__magic_name__				)






						def      __snake_case      (		self  :Tuple			, __magic_name__  :np.ndarray			, __magic_name__  :Union[int, float]			, __magic_name__  :bool = True			, __magic_name__  :Optional[Union[str, ChannelDimension]] = None			, **__magic_name__  :Optional[Any]			, )      ->Any:
										lowercase    :      Optional[int]       = image.astype(np.floataa				)
										if offset:
														lowercase    :      Optional[int]       = image - (scale / 2)
										return rescale(__magic_name__			, scale=__magic_name__			, data_format=__magic_name__			, **__magic_name__				)






						def      __snake_case      (		self  :int			, __magic_name__  :np.ndarray			, __magic_name__  :Union[float, List[float]]			, __magic_name__  :Union[float, List[float]]			, __magic_name__  :Optional[Union[str, ChannelDimension]] = None			, **__magic_name__  :Union[str, Any]			, )      ->np.ndarray:
										return normalize(__magic_name__			, mean=__magic_name__			, std=__magic_name__			, data_format=__magic_name__			, **__magic_name__				)






						def      __snake_case      (		self  :List[Any]			, __magic_name__  :ImageInput			, __magic_name__  :bool = None			, __magic_name__  :Dict[str, int] = None			, __magic_name__  :PILImageResampling = None			, __magic_name__  :bool = None			, __magic_name__  :Dict[str, int] = None			, __magic_name__  :bool = None			, __magic_name__  :float = None			, __magic_name__  :bool = None			, __magic_name__  :bool = None			, __magic_name__  :Optional[Union[float, List[float]]] = None			, __magic_name__  :Optional[Union[float, List[float]]] = None			, __magic_name__  :Optional[ChannelDimension] = ChannelDimension.FIRST			, )      ->np.ndarray:
										if do_resize and size is None or resample is None:
														raise ValueError("""Size and resample must be specified if do_resize is True."""				)

										if do_center_crop and crop_size is None:
														raise ValueError("""Crop size must be specified if do_center_crop is True."""				)

										if do_rescale and rescale_factor is None:
														raise ValueError("""Rescale factor must be specified if do_rescale is True."""				)

										if do_normalize and (image_mean is None or image_std is None):
														raise ValueError("""Image mean and std must be specified if do_normalize is True."""				)

										if offset and not do_rescale:
														raise ValueError("""For offset, do_rescale must also be set to True."""				)

										# All transformations expect numpy arrays.
										lowercase    :      Optional[Any]       = to_numpy_array(__magic_name__				)

										if do_resize:
														lowercase    :      Any       = self.resize(image=__magic_name__			, size=__magic_name__			, resample=__magic_name__				)

										if do_center_crop:
														lowercase    :      Any       = self.center_crop(__magic_name__			, size=__magic_name__				)

										if do_rescale:
														lowercase    :      Optional[int]       = self.rescale(image=__magic_name__			, scale=__magic_name__			, offset=__magic_name__				)

										if do_normalize:
														lowercase    :      Union[str, Any]       = self.normalize(image=__magic_name__			, mean=__magic_name__			, std=__magic_name__				)

										lowercase    :      Optional[Any]       = to_channel_dimension_format(__magic_name__			, __magic_name__				)
										return image






						def      __snake_case      (		self  :Union[str, Any]			, __magic_name__  :ImageInput			, __magic_name__  :bool = None			, __magic_name__  :Dict[str, int] = None			, __magic_name__  :PILImageResampling = None			, __magic_name__  :bool = None			, __magic_name__  :Dict[str, int] = None			, __magic_name__  :bool = None			, __magic_name__  :float = None			, __magic_name__  :bool = None			, __magic_name__  :bool = None			, __magic_name__  :Optional[Union[float, List[float]]] = None			, __magic_name__  :Optional[Union[float, List[float]]] = None			, __magic_name__  :Optional[Union[str, TensorType]] = None			, __magic_name__  :ChannelDimension = ChannelDimension.FIRST			, **__magic_name__  :List[str]			, )      ->PIL.Image.Image:
										lowercase    :      List[Any]       = do_resize if do_resize is not None else self.do_resize
										lowercase    :      Dict       = resample if resample is not None else self.resample
										lowercase    :      Optional[int]       = do_center_crop if do_center_crop is not None else self.do_center_crop
										lowercase    :      List[Any]       = do_rescale if do_rescale is not None else self.do_rescale
										lowercase    :      Tuple       = rescale_factor if rescale_factor is not None else self.rescale_factor
										lowercase    :      Optional[Any]       = offset if offset is not None else self.offset
										lowercase    :      str       = do_normalize if do_normalize is not None else self.do_normalize
										lowercase    :      List[str]       = image_mean if image_mean is not None else self.image_mean
										lowercase    :      Union[str, Any]       = image_std if image_std is not None else self.image_std

										lowercase    :      Optional[int]       = size if size is not None else self.size
										lowercase    :      Optional[Any]       = get_size_dict(__magic_name__			, default_to_square=__magic_name__				)
										lowercase    :      Tuple       = crop_size if crop_size is not None else self.crop_size
										lowercase    :      List[str]       = get_size_dict(__magic_name__			, param_name="""crop_size"""				)

										if not valid_images(__magic_name__				):
														raise ValueError(
														    """Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, """
														    """torch.Tensor, tf.Tensor or jax.ndarray."""				)

										lowercase    :      Tuple       = make_batched(__magic_name__				)

										lowercase    :      List[str]       = [
										    [
										        self._preprocess_image(
										            image=__magic_name__			, do_resize=__magic_name__			, size=__magic_name__			, resample=__magic_name__			, do_center_crop=__magic_name__			, crop_size=__magic_name__			, do_rescale=__magic_name__			, rescale_factor=__magic_name__			, offset=__magic_name__			, do_normalize=__magic_name__			, image_mean=__magic_name__			, image_std=__magic_name__			, data_format=__magic_name__			, )
										        for img in video
										    ]
										    for video in videos
										]

										lowercase    :      Dict       = {"""pixel_values""": videos}
										return BatchFeature(data=__magic_name__			, tensor_type=__magic_name__				)
