
import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_						:     Optional[Any]	       =							get_tests_dir("""fixtures/test_sentencepiece_no_bos.model""")




@require_sentencepiece
@require_tokenizers
class   __lowercase						(						__snake_case   , unittest.TestCase       ):
	_A       =   PegasusTokenizer
	_A       =   PegasusTokenizerFast
	_A       =   True
	_A       =   True






	def       _a(self				: Optional[int]    ) ->			Optional[int]:
							super().setUp()

							# We have a SentencePiece fixture for testing
							_lowercase       :	Any           =   PegasusTokenizer(snake_case    )
							tokenizer.save_pretrained(self.tmpdirname    )






	@cached_property
	def       _a(self				: str    ) ->			Optional[Any]:
							return PegasusTokenizer.from_pretrained("google/pegasus-large"    )






	def       _a(self				: Dict      ,	**snake_case				: List[Any]    ) ->			PegasusTokenizer:
							return PegasusTokenizer.from_pretrained(self.tmpdirname      ,	**snake_case    )






	def       _a(self				: Any      ,	snake_case				: Optional[Any]    ) ->			int:
							return ("This is a test", "This is a test")






	def       _a(self				: Dict    ) ->			Union[str, Any]:
							_lowercase       :	Optional[Any]           =   "</s>"
							_lowercase       :	List[str]           =   1

							self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case    )      ,	snake_case    )
							self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case    )      ,	snake_case    )






	def       _a(self				: Optional[int]    ) ->			Optional[int]:
							_lowercase       :	Union[str, Any]           =   list(self.get_tokenizer().get_vocab().keys()    )

							self.assertEqual(vocab_keys[0]      ,	"<pad>"    )
							self.assertEqual(vocab_keys[1]      ,	"</s>"    )
							self.assertEqual(vocab_keys[-1]      ,	"v"    )
							self.assertEqual(len(snake_case    )      ,	1103    )






	def       _a(self				: List[Any]    ) ->			Dict:
							self.assertEqual(self.get_tokenizer().vocab_size      ,	1103    )






	def       _a(self				: str    ) ->			Dict:
							_lowercase       :	List[str]           =   self.rust_tokenizer_class.from_pretrained(self.tmpdirname    )
							_lowercase       :	List[Any]           =   self.tokenizer_class.from_pretrained(self.tmpdirname    )
							_lowercase       :	Dict           =   (
							    "Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"
							    " </s> <pad> <pad> <pad>"
							)
							_lowercase       :	Union[str, Any]           =   rust_tokenizer([raw_input_str]      ,	return_tensors=snake_case      ,	add_special_tokens=snake_case    ).input_ids[0]
							_lowercase       :	int           =   py_tokenizer([raw_input_str]      ,	return_tensors=snake_case      ,	add_special_tokens=snake_case    ).input_ids[0]
							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: Dict    ) ->			Optional[Any]:
							_lowercase       :	Tuple           =   self._large_tokenizer
							# <mask_1> masks whole sentence while <mask_2> masks single word
							_lowercase       :	Any           =   "<mask_1> To ensure a <mask_2> flow of bank resolutions."
							_lowercase       :	Tuple           =   [2, 413, 615, 114, 3, 1971, 113, 1679, 1_0710, 107, 1]
							_lowercase       :	Any           =   tokenizer([raw_input_str]      ,	return_tensors=snake_case    ).input_ids[0]
							self.assertListEqual(snake_case      ,	snake_case    )






	def       _a(self				: List[Any]    ) ->			List[Any]:
							_lowercase       :	List[str]           =   self._large_tokenizer
							# The tracebacks for the following asserts are **better** without messages or self.assertEqual
							assert tokenizer.vocab_size == 9_6103
							assert tokenizer.pad_token_id == 0
							assert tokenizer.eos_token_id == 1
							assert tokenizer.offset == 103
							assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
							assert tokenizer.unk_token == "<unk>"
							assert tokenizer.model_max_length == 1024
							_lowercase       :	List[str]           =   "To ensure a smooth flow of bank resolutions."
							_lowercase       :	Optional[Any]           =   [413, 615, 114, 2291, 1971, 113, 1679, 1_0710, 107, 1]
							_lowercase       :	Any           =   tokenizer([raw_input_str]      ,	return_tensors=snake_case    ).input_ids[0]
							self.assertListEqual(snake_case      ,	snake_case    )
							assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3]    ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]






	@require_torch
	def       _a(self				: List[Any]    ) ->			Tuple:
							_lowercase       :	int           =   ["This is going to be way too long." * 150, "short example"]
							_lowercase       :	Tuple           =   ["not super long but more than 5 tokens", "tiny"]
							_lowercase       :	Any           =   self._large_tokenizer(snake_case      ,	padding=snake_case      ,	truncation=snake_case      ,	return_tensors="pt"    )
							_lowercase       :	Any           =   self._large_tokenizer(
							    text_target=snake_case      ,	max_length=5      ,	padding=snake_case      ,	truncation=snake_case      ,	return_tensors="pt"    )

							assert batch.input_ids.shape == (2, 1024)
							assert batch.attention_mask.shape == (2, 1024)
							assert targets["input_ids"].shape == (2, 5)
							assert len(snake_case    ) == 2  # input_ids, attention_mask.






	@slow
	def       _a(self				: Union[str, Any]    ) ->			List[Any]:
							# fmt: off
							_lowercase       :	List[Any]           =   {"input_ids": [[3_8979, 143, 1_8485, 606, 130, 2_6669, 8_7686, 121, 5_4189, 1129, 111, 2_6669, 8_7686, 121, 9114, 1_4787, 121, 1_3249, 158, 592, 956, 121, 1_4621, 3_1576, 143, 6_2613, 108, 9688, 930, 4_3430, 1_1562, 6_2613, 304, 108, 1_1443, 897, 108, 9314, 1_7415, 6_3399, 108, 1_1443, 7614, 1_8316, 118, 4284, 7148, 1_2430, 143, 1400, 2_5703, 158, 111, 4284, 7148, 1_1772, 143, 2_1297, 1064, 158, 122, 204, 3506, 1754, 1133, 1_4787, 1581, 115, 3_3224, 4482, 111, 1355, 110, 2_9173, 317, 5_0833, 108, 2_0147, 9_4665, 111, 7_7198, 107, 1], [110, 6_2613, 117, 638, 112, 1133, 121, 2_0098, 1355, 7_9050, 1_3872, 135, 1596, 5_3541, 1352, 141, 1_3039, 5542, 124, 302, 518, 111, 268, 2956, 115, 149, 4427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1235, 2799, 1_8289, 1_7780, 204, 109, 9474, 1296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
							# fmt: on

							self.tokenizer_integration_test_util(
							    expected_encoding=snake_case      ,	model_name="google/bigbird-pegasus-large-arxiv"      ,	revision="ba85d0851d708441f91440d509690f1ab6353415"      ,	)




@require_sentencepiece
@require_tokenizers
class   __lowercase						(						__snake_case   , unittest.TestCase       ):
	_A       =   PegasusTokenizer
	_A       =   PegasusTokenizerFast
	_A       =   True
	_A       =   True






	def       _a(self				: List[str]    ) ->			Any:
							super().setUp()

							# We have a SentencePiece fixture for testing
							_lowercase       :	List[str]           =   PegasusTokenizer(snake_case      ,	offset=0      ,	mask_token_sent=snake_case      ,	mask_token="[MASK]"    )
							tokenizer.save_pretrained(self.tmpdirname    )






	@cached_property
	def       _a(self				: Optional[int]    ) ->			int:
							return PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv"    )






	def       _a(self				: Optional[int]      ,	**snake_case				: Optional[Any]    ) ->			PegasusTokenizer:
							return PegasusTokenizer.from_pretrained(self.tmpdirname      ,	**snake_case    )






	def       _a(self				: Tuple      ,	snake_case				: Union[str, Any]    ) ->			List[str]:
							return ("This is a test", "This is a test")






	def       _a(self				: Any    ) ->			Optional[Any]:
							_lowercase       :	Dict           =   self.rust_tokenizer_class.from_pretrained(self.tmpdirname    )
							_lowercase       :	Union[str, Any]           =   self.tokenizer_class.from_pretrained(self.tmpdirname    )
							_lowercase       :	Tuple           =   (
							    "Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"
							    " <pad> <pad> <pad>"
							)
							_lowercase       :	Optional[Any]           =   rust_tokenizer([raw_input_str]      ,	return_tensors=snake_case      ,	add_special_tokens=snake_case    ).input_ids[0]
							_lowercase       :	Union[str, Any]           =   py_tokenizer([raw_input_str]      ,	return_tensors=snake_case      ,	add_special_tokens=snake_case    ).input_ids[0]
							self.assertListEqual(snake_case      ,	snake_case    )






	@require_torch
	def       _a(self				: Optional[Any]    ) ->			str:
							_lowercase       :	List[str]           =   ["This is going to be way too long." * 1000, "short example"]
							_lowercase       :	Dict           =   ["not super long but more than 5 tokens", "tiny"]
							_lowercase       :	List[Any]           =   self._large_tokenizer(snake_case      ,	padding=snake_case      ,	truncation=snake_case      ,	return_tensors="pt"    )
							_lowercase       :	Any           =   self._large_tokenizer(
							    text_target=snake_case      ,	max_length=5      ,	padding=snake_case      ,	truncation=snake_case      ,	return_tensors="pt"    )

							assert batch.input_ids.shape == (2, 4096)
							assert batch.attention_mask.shape == (2, 4096)
							assert targets["input_ids"].shape == (2, 5)
							assert len(snake_case    ) == 2  # input_ids, attention_mask.






	def       _a(self				: Optional[int]    ) ->			Optional[Any]:
							_lowercase       :	int           =   (
							    "This is an example string that is used to test the original TF implementation against the HF"
							    " implementation"
							)

							_lowercase       :	Union[str, Any]           =   self._large_tokenizer(snake_case    ).input_ids

							self.assertListEqual(
							    snake_case      ,	[182, 117, 142, 587, 4211, 120, 117, 263, 112, 804, 109, 856, 2_5016, 3137, 464, 109, 2_6955, 3137, 1]      ,	)






from typing import Dict, List, Optional, Tuple, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_torch_available, is_torch_tensor, logging


if is_torch_available():
							import torch


UpperCamelCase_						:     Tuple	       =							logging.get_logger(__name__)




class   __lowercase						(						__snake_case       ):
	_A       =   ["pixel_values"]






	def __init__(self				: List[str]      ,	snake_case				: bool = True      ,	snake_case				: Optional[Dict[str, int]] = None      ,	snake_case				: PILImageResampling = PILImageResampling.BILINEAR      ,	snake_case				: bool = True      ,	snake_case				: Dict[str, int] = None      ,	snake_case				: bool = True      ,	snake_case				: Union[int, float] = 1 / 255      ,	snake_case				: bool = True      ,	snake_case				: Optional[Union[float, List[float]]] = None      ,	snake_case				: Optional[Union[float, List[float]]] = None      ,	**snake_case				: List[Any]      ,	) ->			None:
							super().__init__(**snake_case    )
							_lowercase       :	List[str]           =   size if size is not None else {"shortest_edge": 256}
							_lowercase       :	Union[str, Any]           =   get_size_dict(snake_case      ,	default_to_square=snake_case    )
							_lowercase       :	List[str]           =   crop_size if crop_size is not None else {"height": 224, "width": 224}
							_lowercase       :	Dict           =   get_size_dict(snake_case      ,	param_name="crop_size"    )
							_lowercase       :	List[Any]           =   do_resize
							_lowercase       :	Optional[Any]           =   size
							_lowercase       :	Tuple           =   resample
							_lowercase       :	Tuple           =   do_center_crop
							_lowercase       :	Any           =   crop_size
							_lowercase       :	str           =   do_rescale
							_lowercase       :	int           =   rescale_factor
							_lowercase       :	List[Any]           =   do_normalize
							_lowercase       :	List[str]           =   image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
							_lowercase       :	Optional[Any]           =   image_std if image_std is not None else IMAGENET_STANDARD_STD






	def       _a(self				: Tuple      ,	snake_case				: np.ndarray      ,	snake_case				: Dict[str, int]      ,	snake_case				: PILImageResampling = PILImageResampling.BICUBIC      ,	snake_case				: Optional[Union[str, ChannelDimension]] = None      ,	**snake_case				: int      ,	) ->			np.ndarray:
							_lowercase       :	Union[str, Any]           =   get_size_dict(snake_case      ,	default_to_square=snake_case    )
							if "shortest_edge" not in size:
													raise ValueError(F"""The `size` parameter must contain the key `shortest_edge`. Got {size.keys()}"""    )
							_lowercase       :	int           =   get_resize_output_image_size(snake_case      ,	size=size["shortest_edge"]      ,	default_to_square=snake_case    )
							return resize(snake_case      ,	size=snake_case      ,	resample=snake_case      ,	data_format=snake_case      ,	**snake_case    )






	def       _a(self				: str      ,	snake_case				: np.ndarray      ,	snake_case				: Dict[str, int]      ,	snake_case				: Optional[Union[str, ChannelDimension]] = None      ,	**snake_case				: Union[str, Any]      ,	) ->			np.ndarray:
							_lowercase       :	Union[str, Any]           =   get_size_dict(snake_case    )
							if "height" not in size or "width" not in size:
													raise ValueError(F"""The `size` parameter must contain the keys `height` and `width`. Got {size.keys()}"""    )
							return center_crop(snake_case      ,	size=(size["height"], size["width"])      ,	data_format=snake_case      ,	**snake_case    )






	def       _a(self				: Union[str, Any]      ,	snake_case				: np.ndarray      ,	snake_case				: float      ,	snake_case				: Optional[Union[str, ChannelDimension]] = None      ,	**snake_case				: List[Any]    ) ->			np.ndarray:
							return rescale(snake_case      ,	scale=snake_case      ,	data_format=snake_case      ,	**snake_case    )






	def       _a(self				: int      ,	snake_case				: np.ndarray      ,	snake_case				: Union[float, List[float]]      ,	snake_case				: Union[float, List[float]]      ,	snake_case				: Optional[Union[str, ChannelDimension]] = None      ,	**snake_case				: Tuple      ,	) ->			np.ndarray:
							return normalize(snake_case      ,	mean=snake_case      ,	std=snake_case      ,	data_format=snake_case      ,	**snake_case    )






	def       _a(self				: Optional[int]      ,	snake_case				: ImageInput      ,	snake_case				: Optional[bool] = None      ,	snake_case				: Dict[str, int] = None      ,	snake_case				: PILImageResampling = None      ,	snake_case				: bool = None      ,	snake_case				: Dict[str, int] = None      ,	snake_case				: Optional[bool] = None      ,	snake_case				: Optional[float] = None      ,	snake_case				: Optional[bool] = None      ,	snake_case				: Optional[Union[float, List[float]]] = None      ,	snake_case				: Optional[Union[float, List[float]]] = None      ,	snake_case				: Optional[Union[str, TensorType]] = None      ,	snake_case				: Union[str, ChannelDimension] = ChannelDimension.FIRST      ,	**snake_case				: Tuple      ,	) ->			Union[str, Any]:
							_lowercase       :	Any           =   do_resize if do_resize is not None else self.do_resize
							_lowercase       :	List[str]           =   size if size is not None else self.size
							_lowercase       :	Optional[int]           =   get_size_dict(snake_case      ,	default_to_square=snake_case    )
							_lowercase       :	Tuple           =   resample if resample is not None else self.resample
							_lowercase       :	List[Any]           =   do_center_crop if do_center_crop is not None else self.do_center_crop
							_lowercase       :	int           =   crop_size if crop_size is not None else self.crop_size
							_lowercase       :	Dict           =   get_size_dict(snake_case      ,	param_name="crop_size"    )
							_lowercase       :	Dict           =   do_rescale if do_rescale is not None else self.do_rescale
							_lowercase       :	List[Any]           =   rescale_factor if rescale_factor is not None else self.rescale_factor
							_lowercase       :	Union[str, Any]           =   do_normalize if do_normalize is not None else self.do_normalize
							_lowercase       :	Dict           =   image_mean if image_mean is not None else self.image_mean
							_lowercase       :	Optional[Any]           =   image_std if image_std is not None else self.image_std

							_lowercase       :	int           =   make_list_of_images(snake_case    )

							if not valid_images(snake_case    ):
													raise ValueError(
													    "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, "
													    "torch.Tensor, tf.Tensor or jax.ndarray."    )

							if do_resize and size is None:
													raise ValueError("Size must be specified if do_resize is True."    )

							if do_center_crop and crop_size is None:
													raise ValueError("Crop size must be specified if do_center_crop is True."    )

							if do_rescale and rescale_factor is None:
													raise ValueError("Rescale factor must be specified if do_rescale is True."    )

							if do_normalize and (image_mean is None or image_std is None):
													raise ValueError("Image mean and std must be specified if do_normalize is True."    )

							# All transformations expect numpy arrays.
							_lowercase       :	Tuple           =   [to_numpy_array(snake_case    ) for image in images]

							if do_resize:
													_lowercase       :	List[str]           =   [self.resize(image=snake_case      ,	size=snake_case      ,	resample=snake_case    ) for image in images]

							if do_center_crop:
													_lowercase       :	List[str]           =   [self.center_crop(image=snake_case      ,	size=snake_case    ) for image in images]

							if do_rescale:
													_lowercase       :	Any           =   [self.rescale(image=snake_case      ,	scale=snake_case    ) for image in images]

							if do_normalize:
													_lowercase       :	Optional[Any]           =   [self.normalize(image=snake_case      ,	mean=snake_case      ,	std=snake_case    ) for image in images]

							_lowercase       :	Any           =   [to_channel_dimension_format(snake_case      ,	snake_case    ) for image in images]

							_lowercase       :	Optional[Any]           =   {"pixel_values": images}
							return BatchFeature(data=snake_case      ,	tensor_type=snake_case    )






	def       _a(self				: Dict      ,	snake_case				: List[str]      ,	snake_case				: List[Tuple] = None    ) ->			Optional[Any]:
							_lowercase       :	Dict           =   outputs.logits

							# Resize logits and compute semantic segmentation maps
							if target_sizes is not None:
													if len(snake_case    ) != len(snake_case    ):
																			raise ValueError(
																			    "Make sure that you pass in as many target sizes as the batch dimension of the logits"    )

													if is_torch_tensor(snake_case    ):
																			_lowercase       :	Dict           =   target_sizes.numpy()

													_lowercase       :	Tuple           =   []

													for idx in range(len(snake_case    )    ):
																			_lowercase       :	List[str]           =   torch.nn.functional.interpolate(
																			    logits[idx].unsqueeze(dim=0    )      ,	size=target_sizes[idx]      ,	mode="bilinear"      ,	align_corners=snake_case    )
																			_lowercase       :	int           =   resized_logits[0].argmax(dim=0    )
																			semantic_segmentation.append(snake_case    )
							else:
													_lowercase       :	Optional[int]           =   logits.argmax(dim=1    )
													_lowercase       :	int           =   [semantic_segmentation[i] for i in range(semantic_segmentation.shape[0]    )]

							return semantic_segmentation




