
import unittest

from transformers import PegasusTokenizer, PegasusTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


UpperCamelCase_   	=	get_tests_dir("fixtures/test_sentencepiece_no_bos.model")

@require_sentencepiece
@require_tokenizers
class       a		(   __UpperCAmelCase      ,	unittest.TestCase					):
							lowercase_		:						Optional[Any]   							=  PegasusTokenizer
							lowercase_		:						str   							=  PegasusTokenizerFast
							lowercase_		:						Optional[Any]   							=  True
							lowercase_		:						Optional[int]   							=  True







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								super().setUp()

								# We have a SentencePiece fixture for testing
								__lowerCAmelCase         =							PegasusTokenizer(snake_case__ )
								tokenizer.save_pretrained(self.tmpdirname )







							@cached_property
							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								return PegasusTokenizer.from_pretrained("google/pegasus-large" )







							def       UpperCAmelCase__					(						self  :  Dict      ,				**snake_case__  :  Tuple ):
								"""simple docstring"""







								return PegasusTokenizer.from_pretrained(self.tmpdirname      ,				**snake_case__ )







							def       UpperCAmelCase__					(						self  :  Optional[int]      ,				snake_case__  :  Tuple ):
								"""simple docstring"""







								return ("This is a test", "This is a test")







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							"</s>"
								__lowerCAmelCase         =							1

								self.assertEqual(self.get_tokenizer()._convert_token_to_id(snake_case__ )      ,				snake_case__ )
								self.assertEqual(self.get_tokenizer()._convert_id_to_token(snake_case__ )      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							list(self.get_tokenizer().get_vocab().keys() )

								self.assertEqual(vocab_keys[0]      ,				"<pad>" )
								self.assertEqual(vocab_keys[1]      ,				"</s>" )
								self.assertEqual(vocab_keys[-1]      ,				"v" )
								self.assertEqual(len(snake_case__ )      ,				1_103 )







							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								self.assertEqual(self.get_tokenizer().vocab_size      ,				1_103 )







							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							self.rust_tokenizer_class.from_pretrained(self.tmpdirname )
								__lowerCAmelCase         =							self.tokenizer_class.from_pretrained(self.tmpdirname )
								__lowerCAmelCase         =							(
								    "Let's see which <unk> is the better <unk_token_11> one <mask_1> It seems like this <mask_2> was important"
								    " </s> <pad> <pad> <pad>"
								)
								__lowerCAmelCase         =							rust_tokenizer([raw_input_str]      ,				return_tensors=snake_case__      ,				add_special_tokens=snake_case__ ).input_ids[0]
								__lowerCAmelCase         =							py_tokenizer([raw_input_str]      ,				return_tensors=snake_case__      ,				add_special_tokens=snake_case__ ).input_ids[0]
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  str ):
								"""simple docstring"""







								__lowerCAmelCase         =							self._large_tokenizer
								# <mask_1> masks whole sentence while <mask_2> masks single word
								__lowerCAmelCase         =							"<mask_1> To ensure a <mask_2> flow of bank resolutions."
								__lowerCAmelCase         =							[2, 413, 615, 114, 3, 1_971, 113, 1_679, 10_710, 107, 1]
								__lowerCAmelCase         =							tokenizer([raw_input_str]      ,				return_tensors=snake_case__ ).input_ids[0]
								self.assertListEqual(snake_case__      ,				snake_case__ )







							def       UpperCAmelCase__					(						self  :  Tuple ):
								"""simple docstring"""







								__lowerCAmelCase         =							self._large_tokenizer
								# The tracebacks for the following asserts are **better** without messages or self.assertEqual
								assert tokenizer.vocab_size == 96_103
								assert tokenizer.pad_token_id == 0
								assert tokenizer.eos_token_id == 1
								assert tokenizer.offset == 103
								assert tokenizer.unk_token_id == tokenizer.offset + 2 == 105
								assert tokenizer.unk_token == "<unk>"
								assert tokenizer.model_max_length == 1_024
								__lowerCAmelCase         =							"To ensure a smooth flow of bank resolutions."
								__lowerCAmelCase         =							[413, 615, 114, 2_291, 1_971, 113, 1_679, 10_710, 107, 1]
								__lowerCAmelCase         =							tokenizer([raw_input_str]      ,				return_tensors=snake_case__ ).input_ids[0]
								self.assertListEqual(snake_case__      ,				snake_case__ )
								assert tokenizer.convert_ids_to_tokens([0, 1, 2, 3] ) == ["<pad>", "</s>", "<mask_1>", "<mask_2>"]







							@require_torch
							def       UpperCAmelCase__					(						self  :  Any ):
								"""simple docstring"""







								__lowerCAmelCase         =							["This is going to be way too long." * 150, "short example"]
								__lowerCAmelCase         =							["not super long but more than 5 tokens", "tiny"]
								__lowerCAmelCase         =							self._large_tokenizer(snake_case__      ,				padding=snake_case__      ,				truncation=snake_case__      ,				return_tensors="pt" )
								__lowerCAmelCase         =							self._large_tokenizer(
								    text_target=snake_case__      ,				max_length=5      ,				padding=snake_case__      ,				truncation=snake_case__      ,				return_tensors="pt" )

								assert batch.input_ids.shape == (2, 1_024)
								assert batch.attention_mask.shape == (2, 1_024)
								assert targets["input_ids"].shape == (2, 5)
								assert len(snake_case__ ) == 2  # input_ids, attention_mask.







							@slow
							def       UpperCAmelCase__					(						self  :  Optional[Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							{"input_ids": [[38_979, 143, 18_485, 606, 130, 26_669, 87_686, 121, 54_189, 1_129, 111, 26_669, 87_686, 121, 9_114, 14_787, 121, 13_249, 158, 592, 956, 121, 14_621, 31_576, 143, 62_613, 108, 9_688, 930, 43_430, 11_562, 62_613, 304, 108, 11_443, 897, 108, 9_314, 17_415, 63_399, 108, 11_443, 7_614, 18_316, 118, 4_284, 7_148, 12_430, 143, 1_400, 25_703, 158, 111, 4_284, 7_148, 11_772, 143, 21_297, 1_064, 158, 122, 204, 3_506, 1_754, 1_133, 14_787, 1_581, 115, 33_224, 4_482, 111, 1_355, 110, 29_173, 317, 50_833, 108, 20_147, 94_665, 111, 77_198, 107, 1], [110, 62_613, 117, 638, 112, 1_133, 121, 20_098, 1_355, 79_050, 13_872, 135, 1_596, 53_541, 1_352, 141, 13_039, 5_542, 124, 302, 518, 111, 268, 2_956, 115, 149, 4_427, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [139, 1_235, 2_799, 18_289, 17_780, 204, 109, 9_474, 1_296, 107, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], "attention_mask": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
								# fmt: on

								self.tokenizer_integration_test_util(
								    expected_encoding=snake_case__      ,				model_name="google/bigbird-pegasus-large-arxiv"      ,				revision="ba85d0851d708441f91440d509690f1ab6353415"      ,				)

@require_sentencepiece
@require_tokenizers
class       a		(   __UpperCAmelCase      ,	unittest.TestCase					):
							lowercase_		:						List[str]   							=  PegasusTokenizer
							lowercase_		:						List[Any]   							=  PegasusTokenizerFast
							lowercase_		:						Any   							=  True
							lowercase_		:						int   							=  True







							def       UpperCAmelCase__					(						self  :  List[Any] ):
								"""simple docstring"""







								super().setUp()

								# We have a SentencePiece fixture for testing
								__lowerCAmelCase         =							PegasusTokenizer(snake_case__      ,				offset=0      ,				mask_token_sent=snake_case__      ,				mask_token="[MASK]" )
								tokenizer.save_pretrained(self.tmpdirname )







							@cached_property
							def       UpperCAmelCase__					(						self  :  Optional[Any] ):
								"""simple docstring"""







								return PegasusTokenizer.from_pretrained("google/bigbird-pegasus-large-arxiv" )







							def       UpperCAmelCase__					(						self  :  Optional[int]      ,				**snake_case__  :  Optional[int] ):
								"""simple docstring"""







								return PegasusTokenizer.from_pretrained(self.tmpdirname      ,				**snake_case__ )







							def       UpperCAmelCase__					(						self  :  Union[str, Any]      ,				snake_case__  :  Any ):
								"""simple docstring"""







								return ("This is a test", "This is a test")







							def       UpperCAmelCase__					(						self  :  List[str] ):
								"""simple docstring"""







								__lowerCAmelCase         =							self.rust_tokenizer_class.from_pretrained(self.tmpdirname )
								__lowerCAmelCase         =							self.tokenizer_class.from_pretrained(self.tmpdirname )
								__lowerCAmelCase         =							(
								    "Let's see which <unk> is the better <unk_token> one [MASK] It seems like this [MASK] was important </s>"
								    " <pad> <pad> <pad>"
								)
								__lowerCAmelCase         =							rust_tokenizer([raw_input_str]      ,				return_tensors=snake_case__      ,				add_special_tokens=snake_case__ ).input_ids[0]
								__lowerCAmelCase         =							py_tokenizer([raw_input_str]      ,				return_tensors=snake_case__      ,				add_special_tokens=snake_case__ ).input_ids[0]
								self.assertListEqual(snake_case__      ,				snake_case__ )







							@require_torch
							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							["This is going to be way too long." * 1_000, "short example"]
								__lowerCAmelCase         =							["not super long but more than 5 tokens", "tiny"]
								__lowerCAmelCase         =							self._large_tokenizer(snake_case__      ,				padding=snake_case__      ,				truncation=snake_case__      ,				return_tensors="pt" )
								__lowerCAmelCase         =							self._large_tokenizer(
								    text_target=snake_case__      ,				max_length=5      ,				padding=snake_case__      ,				truncation=snake_case__      ,				return_tensors="pt" )

								assert batch.input_ids.shape == (2, 4_096)
								assert batch.attention_mask.shape == (2, 4_096)
								assert targets["input_ids"].shape == (2, 5)
								assert len(snake_case__ ) == 2  # input_ids, attention_mask.







							def       UpperCAmelCase__					(						self  :  Optional[int] ):
								"""simple docstring"""







								__lowerCAmelCase         =							(
								    "This is an example string that is used to test the original TF implementation against the HF"
								    " implementation"
								)

								__lowerCAmelCase         =							self._large_tokenizer(snake_case__ ).input_ids

								self.assertListEqual(
								    snake_case__      ,				[182, 117, 142, 587, 4_211, 120, 117, 263, 112, 804, 109, 856, 25_016, 3_137, 464, 109, 26_955, 3_137, 1]      ,				)








from copy import deepcopy
from typing import Optional, Union

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import TensorType, is_tf_available, is_torch_available


if is_torch_available():
					import torch

if is_tf_available():
					import tensorflow as tf

class       a		(   __UpperCAmelCase					):
							lowercase_		:						Tuple   							=  ['image_processor']
							lowercase_		:						Tuple   							=  'SamImageProcessor'







							def __init__(						self  :  List[str]      ,				snake_case__  :  Optional[Any] ):
								"""simple docstring"""







								super().__init__(snake_case__ )
								__lowerCAmelCase         =							self.image_processor
								__lowerCAmelCase         =							-10
								__lowerCAmelCase         =							self.image_processor.size["longest_edge"]







							def __call__(						self  :  Any      ,				snake_case__  :  Optional[int]=None      ,				snake_case__  :  Optional[int]=None      ,				snake_case__  :  str=None      ,				snake_case__  :  List[str]=None      ,				snake_case__  :  Optional[Union[str, TensorType]] = None      ,				**snake_case__  :  Any      ,				):
								"""simple docstring"""







								__lowerCAmelCase         =							self.image_processor(
								    snake_case__      ,				return_tensors=snake_case__      ,				**snake_case__      ,				)

								# pop arguments that are not used in the foward but used nevertheless
								__lowerCAmelCase         =							encoding_image_processor["original_sizes"]

								if hasattr(snake_case__      ,				"numpy" ):  # Checks if Torch or TF tensor
									__lowerCAmelCase         =							original_sizes.numpy()

								__lowerCAmelCase   ,						__lowerCAmelCase   ,						__lowerCAmelCase         =							self._check_and_preprocess_points(
								    input_points=snake_case__      ,				input_labels=snake_case__      ,				input_boxes=snake_case__      ,				)

								__lowerCAmelCase         =							self._normalize_and_convert(
								    snake_case__      ,				snake_case__      ,				input_points=snake_case__      ,				input_labels=snake_case__      ,				input_boxes=snake_case__      ,				return_tensors=snake_case__      ,				)

								return encoding_image_processor







							def       UpperCAmelCase__					(						self  :  Tuple      ,				snake_case__  :  Any      ,				snake_case__  :  Optional[Any]      ,				snake_case__  :  str=None      ,				snake_case__  :  Any=None      ,				snake_case__  :  Any=None      ,				snake_case__  :  List[str]="pt"      ,				):
								"""simple docstring"""







								if input_points is not None:
									if len(snake_case__ ) != len(snake_case__ ):
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				original_sizes[0] ) for point in input_points
										]
									else:
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				snake_case__ )
										    for point, original_size in zip(snake_case__      ,				snake_case__ )
										]
									# check that all arrays have the same shape
									if not all(point.shape == input_points[0].shape for point in input_points ):
										if input_labels is not None:
											__lowerCAmelCase   ,						__lowerCAmelCase         =							self._pad_points_and_labels(snake_case__      ,				snake_case__ )

									__lowerCAmelCase         =							np.array(snake_case__ )

								if input_labels is not None:
									__lowerCAmelCase         =							np.array(snake_case__ )

								if input_boxes is not None:
									if len(snake_case__ ) != len(snake_case__ ):
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				original_sizes[0]      ,				is_bounding_box=snake_case__ )
										    for box in input_boxes
										]
									else:
										__lowerCAmelCase         =							[
										    self._normalize_coordinates(self.target_size      ,				snake_case__      ,				snake_case__      ,				is_bounding_box=snake_case__ )
										    for box, original_size in zip(snake_case__      ,				snake_case__ )
										]
									__lowerCAmelCase         =							np.array(snake_case__ )

								if input_boxes is not None:
									if return_tensors == "pt":
										__lowerCAmelCase         =							torch.from_numpy(snake_case__ )
										# boxes batch size of 1 by default
										__lowerCAmelCase         =							input_boxes.unsqueeze(1 ) if len(input_boxes.shape ) != 3 else input_boxes
									elif return_tensors == "tf":
										__lowerCAmelCase         =							tf.convert_to_tensor(snake_case__ )
										# boxes batch size of 1 by default
										__lowerCAmelCase         =							tf.expand_dims(snake_case__      ,				1 ) if len(input_boxes.shape ) != 3 else input_boxes
									encoding_image_processor.update({"input_boxes": input_boxes} )
								if input_points is not None:
									if return_tensors == "pt":
										__lowerCAmelCase         =							torch.from_numpy(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							input_points.unsqueeze(1 ) if len(input_points.shape ) != 4 else input_points
									elif return_tensors == "tf":
										__lowerCAmelCase         =							tf.convert_to_tensor(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							tf.expand_dims(snake_case__      ,				1 ) if len(input_points.shape ) != 4 else input_points
									encoding_image_processor.update({"input_points": input_points} )
								if input_labels is not None:
									if return_tensors == "pt":
										__lowerCAmelCase         =							torch.from_numpy(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							input_labels.unsqueeze(1 ) if len(input_labels.shape ) != 3 else input_labels
									elif return_tensors == "tf":
										__lowerCAmelCase         =							tf.convert_to_tensor(snake_case__ )
										# point batch size of 1 by default
										__lowerCAmelCase         =							tf.expand_dims(snake_case__      ,				1 ) if len(input_labels.shape ) != 3 else input_labels
									encoding_image_processor.update({"input_labels": input_labels} )

								return encoding_image_processor







							def       UpperCAmelCase__					(						self  :  str      ,				snake_case__  :  Union[str, Any]      ,				snake_case__  :  Dict ):
								"""simple docstring"""







								__lowerCAmelCase         =							max([point.shape[0] for point in input_points] )
								__lowerCAmelCase         =							[]
								for i, point in enumerate(snake_case__ ):
									if point.shape[0] != expected_nb_points:
										__lowerCAmelCase         =							np.concatenate(
										    [point, np.zeros((expected_nb_points - point.shape[0], 2) ) + self.point_pad_value]      ,				axis=0 )
										__lowerCAmelCase         =							np.append(input_labels[i]      ,				[self.point_pad_value] )
									processed_input_points.append(snake_case__ )
								__lowerCAmelCase         =							processed_input_points
								return input_points, input_labels







							def       UpperCAmelCase__					(						self  :  Dict      ,				snake_case__  :  int      ,				snake_case__  :  np.ndarray      ,				snake_case__  :  List[str]      ,				snake_case__  :  Dict=False ):
								"""simple docstring"""







								__lowerCAmelCase   ,						__lowerCAmelCase         =							original_size
								__lowerCAmelCase   ,						__lowerCAmelCase         =							self.image_processor._get_preprocess_shape(snake_case__      ,				longest_edge=snake_case__ )
								__lowerCAmelCase         =							deepcopy(snake_case__ ).astype(snake_case__ )

								if is_bounding_box:
									__lowerCAmelCase         =							coords.reshape(-1      ,				2      ,				2 )

								__lowerCAmelCase         =							coords[..., 0] * (new_w / old_w)
								__lowerCAmelCase         =							coords[..., 1] * (new_h / old_h)

								if is_bounding_box:
									__lowerCAmelCase         =							coords.reshape(-1      ,				4 )

								return coords







							def       UpperCAmelCase__					(						self  :  List[str]      ,				snake_case__  :  Union[str, Any]=None      ,				snake_case__  :  int=None      ,				snake_case__  :  Any=None      ,				):
								"""simple docstring"""







								if input_points is not None:
									if hasattr(snake_case__      ,				"numpy" ):  # Checks for TF or Torch tensor
										__lowerCAmelCase         =							input_points.numpy().tolist()

									if not isinstance(snake_case__      ,				snake_case__ ) or not isinstance(input_points[0]      ,				snake_case__ ):
										raise ValueError("Input points must be a list of list of floating points." )
									__lowerCAmelCase         =							[np.array(snake_case__ ) for input_point in input_points]
								else:
									__lowerCAmelCase         =							None

								if input_labels is not None:
									if hasattr(snake_case__      ,				"numpy" ):
										__lowerCAmelCase         =							input_labels.numpy().tolist()

									if not isinstance(snake_case__      ,				snake_case__ ) or not isinstance(input_labels[0]      ,				snake_case__ ):
										raise ValueError("Input labels must be a list of list integers." )
									__lowerCAmelCase         =							[np.array(snake_case__ ) for label in input_labels]
								else:
									__lowerCAmelCase         =							None

								if input_boxes is not None:
									if hasattr(snake_case__      ,				"numpy" ):
										__lowerCAmelCase         =							input_boxes.numpy().tolist()

									if (
									    not isinstance(snake_case__      ,				snake_case__ )
									    or not isinstance(input_boxes[0]      ,				snake_case__ )
									    or not isinstance(input_boxes[0][0]      ,				snake_case__ )
									):
										raise ValueError("Input boxes must be a list of list of list of floating points." )
									__lowerCAmelCase         =							[np.array(snake_case__ ).astype(np.floataa ) for box in input_boxes]
								else:
									__lowerCAmelCase         =							None

								return input_points, input_labels, input_boxes







							@property
							def       UpperCAmelCase__					(						self  :  Union[str, Any] ):
								"""simple docstring"""







								__lowerCAmelCase         =							self.image_processor.model_input_names
								return list(dict.fromkeys(snake_case__ ) )







							def       UpperCAmelCase__					(						self  :  Optional[Any]      ,				*snake_case__  :  Dict      ,				**snake_case__  :  Union[str, Any] ):
								"""simple docstring"""







								return self.image_processor.post_process_masks(*snake_case__      ,				**snake_case__ )






