





'''simple docstring'''




import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_ta import TaTokenizer
else:
							__UpperCAmelCase            =       None


__UpperCAmelCase            =       logging.get_logger(__name__)

__UpperCAmelCase            =       {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

__UpperCAmelCase            =       {
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/tokenizer.json""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/tokenizer.json""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/tokenizer.json""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/tokenizer.json""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/tokenizer.json""",
    },
}


# TODO(PVP) - this should be removed in Transformers v5
__UpperCAmelCase            =       {
    """t5-small""": 512,
    """t5-base""": 512,
    """t5-large""": 512,
    """t5-3b""": 512,
    """t5-11b""": 512,
}




class      UpperCamelCase__					(					lowercase_			):




					"""simple docstring"""
					SCREAMING_SNAKE_CASE__				=	VOCAB_FILES_NAMES
					SCREAMING_SNAKE_CASE__				=	PRETRAINED_VOCAB_FILES_MAP
					SCREAMING_SNAKE_CASE__				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					SCREAMING_SNAKE_CASE__				=	['''input_ids''', '''attention_mask''']
					SCREAMING_SNAKE_CASE__				=	TaTokenizer

					SCREAMING_SNAKE_CASE__				=	[]


					def __init__(						self				:			Dict      ,						lowerCamelCase_				:			Optional[Any]=None      ,						lowerCamelCase_				:			Optional[Any]=None      ,						lowerCamelCase_				:			Any="</s>"      ,						lowerCamelCase_				:			Optional[int]="<unk>"      ,						lowerCamelCase_				:			List[str]="<pad>"      ,						lowerCamelCase_				:			Dict=1_00      ,						lowerCamelCase_				:			List[str]=None      ,						**lowerCamelCase_				:			Tuple      ,						):

							'''simple docstring'''



							if extra_ids > 0 and additional_special_tokens is None:
									SCREAMING_SNAKE_CASE :				Any		=			[f'''<extra_id_{i}>''' for i in range(lowerCamelCase_						)]
							elif extra_ids > 0 and additional_special_tokens is not None:
									# Check that we have the right number of extra special tokens
									SCREAMING_SNAKE_CASE :				Any		=			len(set(filter(lambda lowerCamelCase_						: bool("""extra_id_""" in str(lowerCamelCase_						)						)      ,						lowerCamelCase_						)						)						)
									if extra_tokens != extra_ids:
											raise ValueError(
											    f'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
											    """ provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids"""
											    """ tokens"""						)

							super().__init__(
							    lowerCamelCase_      ,						tokenizer_file=lowerCamelCase_      ,						eos_token=lowerCamelCase_      ,						unk_token=lowerCamelCase_      ,						pad_token=lowerCamelCase_      ,						extra_ids=lowerCamelCase_      ,						additional_special_tokens=lowerCamelCase_      ,						**lowerCamelCase_      ,						)

							SCREAMING_SNAKE_CASE :				str		=			vocab_file
							SCREAMING_SNAKE_CASE :				Optional[int]		=			False if not self.vocab_file else True
							SCREAMING_SNAKE_CASE :				Any		=			extra_ids


					@staticmethod
					def lowerCamelCase_			(						lowerCamelCase_				:			Dict      ,						lowerCamelCase_				:			Optional[int]      ,						lowerCamelCase_				:			List[str]						):

							'''simple docstring'''



							if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
									SCREAMING_SNAKE_CASE :				Union[str, Any]		=			TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
									if init_max_model_length is not None and init_max_model_length != max_model_length:
											return init_max_model_length
									elif init_max_model_length is None:
											warnings.warn(
											    """This tokenizer was incorrectly instantiated with a model max length of"""
											    f''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
											    """ behavior is kept to avoid breaking backwards compatibility when padding/encoding with"""
											    """ `truncation is True`.\n- Be aware that you SHOULD NOT rely on"""
											    f''' {pretrained_model_name_or_path} automatically truncating your input to'''
											    f''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
											    f''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
											    """ `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please"""
											    """ instantiate this tokenizer with `model_max_length` set to your preferred value."""      ,						lowerCamelCase_      ,						)

							return max_model_length


					def lowerCamelCase_			(						self				:			Tuple      ,						lowerCamelCase_				:			str      ,						lowerCamelCase_				:			Optional[str] = None						):

							'''simple docstring'''



							if not self.can_save_slow_tokenizer:
									raise ValueError(
									    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
									    """tokenizer."""						)

							if not os.path.isdir(lowerCamelCase_						):
									logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''						)
									return
							SCREAMING_SNAKE_CASE :				List[str]		=			os.path.join(
							    lowerCamelCase_      ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

							if os.path.abspath(self.vocab_file						) != os.path.abspath(lowerCamelCase_						):
									copyfile(self.vocab_file      ,						lowerCamelCase_						)
									logger.info(f'''Copy vocab file to {out_vocab_file}'''						)

							return (out_vocab_file,)


					def lowerCamelCase_			(						self				:			int      ,						lowerCamelCase_				:			List[int]      ,						lowerCamelCase_				:			Optional[List[int]] = None						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			token_ids_a + [self.eos_token_id]
							if token_ids_a is None:
									return self.prefix_tokens + token_ids_a
							else:
									SCREAMING_SNAKE_CASE :				Optional[Any]		=			token_ids_a + [self.eos_token_id]
									return self.prefix_tokens + token_ids_a + token_ids_a


					def lowerCamelCase_			(						self				:			List[Any]      ,						lowerCamelCase_				:			List[int]      ,						lowerCamelCase_				:			Optional[List[int]] = None						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			[self.eos_token_id]

							if token_ids_a is None:
									return len(token_ids_a + eos						) * [0]
							return len(token_ids_a + eos + token_ids_a + eos						) * [0]


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							return list(
							    set(filter(lambda lowerCamelCase_						: bool(re.search(R"""<extra_id_\d+>"""      ,						lowerCamelCase_						)						) is not None      ,						self.additional_special_tokens						)						)						)



					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							return [self.convert_tokens_to_ids(lowerCamelCase_						) for token in self.get_sentinel_tokens()]










'''simple docstring'''




import shutil
import tempfile
import unittest

import numpy as np

from transformers.testing_utils import (
    is_pt_tf_cross_test,
    require_tf,
    require_torch,
    require_torchvision,
    require_vision,
)
from transformers.utils import is_tf_available, is_torch_available, is_vision_available


if is_vision_available():
							from PIL import Image

							from transformers import AutoProcessor, SamImageProcessor, SamProcessor

if is_torch_available():
							import torch

if is_tf_available():
							import tensorflow as tf




@require_vision
@require_torchvision
class      UpperCamelCase__					(					unittest.TestCase			):




					"""simple docstring"""


					def lowerCamelCase_			(						self				:			Any						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			tempfile.mkdtemp()
							SCREAMING_SNAKE_CASE :				List[Any]		=			SamImageProcessor()
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			SamProcessor(lowerCamelCase_						)
							processor.save_pretrained(self.tmpdirname						)


					def lowerCamelCase_			(						self				:			Dict      ,						**lowerCamelCase_				:			List[str]						):

							'''simple docstring'''



							return AutoProcessor.from_pretrained(self.tmpdirname      ,						**lowerCamelCase_						).image_processor


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							shutil.rmtree(self.tmpdirname						)


					def lowerCamelCase_			(						self				:			str						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			[np.random.randint(2_55      ,						size=(3, 30, 4_00)      ,						dtype=np.uinta						)]

							SCREAMING_SNAKE_CASE :				Optional[int]		=			[Image.fromarray(np.moveaxis(lowerCamelCase_      ,						0      ,						-1						)						) for x in image_inputs]

							return image_inputs


					def lowerCamelCase_			(						self				:			Any						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[Any]		=			SamProcessor(image_processor=self.get_image_processor()						)
							processor.save_pretrained(self.tmpdirname						)

							SCREAMING_SNAKE_CASE :				int		=			self.get_image_processor(do_normalize=lowerCamelCase_      ,						padding_value=1.0						)

							SCREAMING_SNAKE_CASE :				Tuple		=			SamProcessor.from_pretrained(self.tmpdirname      ,						do_normalize=lowerCamelCase_      ,						padding_value=1.0						)

							self.assertEqual(processor.image_processor.to_json_string()      ,						image_processor_add_kwargs.to_json_string()						)
							self.assertIsInstance(processor.image_processor      ,						lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.get_image_processor()

							SCREAMING_SNAKE_CASE :				List[Any]		=			SamProcessor(image_processor=lowerCamelCase_						)

							SCREAMING_SNAKE_CASE :				Optional[int]		=			self.prepare_image_inputs()

							SCREAMING_SNAKE_CASE :				List[str]		=			image_processor(lowerCamelCase_      ,						return_tensors="""np"""						)
							SCREAMING_SNAKE_CASE :				Dict		=			processor(images=lowerCamelCase_      ,						return_tensors="""np"""						)

							input_feat_extract.pop("""original_sizes"""						)  # pop original_sizes as it is popped in the processor
							input_feat_extract.pop("""reshaped_input_sizes"""						)  # pop original_sizes as it is popped in the processor

							for key in input_feat_extract.keys():
									self.assertAlmostEqual(input_feat_extract[key].sum()      ,						input_processor[key].sum()      ,						delta=1e-2						)



					@require_torch
					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				List[str]		=			self.get_image_processor()

							SCREAMING_SNAKE_CASE :				Optional[Any]		=			SamProcessor(image_processor=lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				Any		=			[torch.ones((1, 3, 5, 5)						)]

							SCREAMING_SNAKE_CASE :				int		=			[[17_64, 26_46]]

							SCREAMING_SNAKE_CASE :				Any		=			[[6_83, 10_24]]
							SCREAMING_SNAKE_CASE :				str		=			processor.post_process_masks(lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_						)
							self.assertEqual(masks[0].shape      ,						(1, 3, 17_64, 26_46)						)

							SCREAMING_SNAKE_CASE :				str		=			processor.post_process_masks(
							    lowerCamelCase_      ,						torch.tensor(lowerCamelCase_						)      ,						torch.tensor(lowerCamelCase_						)						)
							self.assertEqual(masks[0].shape      ,						(1, 3, 17_64, 26_46)						)

							# should also work with np
							SCREAMING_SNAKE_CASE :				Optional[int]		=			[np.ones((1, 3, 5, 5)						)]
							SCREAMING_SNAKE_CASE :				Optional[int]		=			processor.post_process_masks(lowerCamelCase_      ,						np.array(lowerCamelCase_						)      ,						np.array(lowerCamelCase_						)						)

							self.assertEqual(masks[0].shape      ,						(1, 3, 17_64, 26_46)						)

							SCREAMING_SNAKE_CASE :				Tuple		=			[[1, 0], [0, 1]]
							with self.assertRaises(lowerCamelCase_						):
									SCREAMING_SNAKE_CASE :				Tuple		=			processor.post_process_masks(lowerCamelCase_      ,						np.array(lowerCamelCase_						)      ,						np.array(lowerCamelCase_						)						)







@require_vision
@require_tf
class      UpperCamelCase__					(					unittest.TestCase			):




					"""simple docstring"""


					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			tempfile.mkdtemp()
							SCREAMING_SNAKE_CASE :				Optional[Any]		=			SamImageProcessor()
							SCREAMING_SNAKE_CASE :				Optional[int]		=			SamProcessor(lowerCamelCase_						)
							processor.save_pretrained(self.tmpdirname						)


					def lowerCamelCase_			(						self				:			Any      ,						**lowerCamelCase_				:			str						):

							'''simple docstring'''



							return AutoProcessor.from_pretrained(self.tmpdirname      ,						**lowerCamelCase_						).image_processor


					def lowerCamelCase_			(						self				:			List[Any]						):

							'''simple docstring'''



							shutil.rmtree(self.tmpdirname						)


					def lowerCamelCase_			(						self				:			Union[str, Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Any		=			[np.random.randint(2_55      ,						size=(3, 30, 4_00)      ,						dtype=np.uinta						)]

							SCREAMING_SNAKE_CASE :				str		=			[Image.fromarray(np.moveaxis(lowerCamelCase_      ,						0      ,						-1						)						) for x in image_inputs]

							return image_inputs


					def lowerCamelCase_			(						self				:			List[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			SamProcessor(image_processor=self.get_image_processor()						)
							processor.save_pretrained(self.tmpdirname						)

							SCREAMING_SNAKE_CASE :				Optional[Any]		=			self.get_image_processor(do_normalize=lowerCamelCase_      ,						padding_value=1.0						)

							SCREAMING_SNAKE_CASE :				List[Any]		=			SamProcessor.from_pretrained(self.tmpdirname      ,						do_normalize=lowerCamelCase_      ,						padding_value=1.0						)

							self.assertEqual(processor.image_processor.to_json_string()      ,						image_processor_add_kwargs.to_json_string()						)
							self.assertIsInstance(processor.image_processor      ,						lowerCamelCase_						)


					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				str		=			self.get_image_processor()

							SCREAMING_SNAKE_CASE :				Dict		=			SamProcessor(image_processor=lowerCamelCase_						)

							SCREAMING_SNAKE_CASE :				int		=			self.prepare_image_inputs()

							SCREAMING_SNAKE_CASE :				Optional[Any]		=			image_processor(lowerCamelCase_      ,						return_tensors="""np"""						)
							SCREAMING_SNAKE_CASE :				List[Any]		=			processor(images=lowerCamelCase_      ,						return_tensors="""np"""						)

							input_feat_extract.pop("""original_sizes"""						)  # pop original_sizes as it is popped in the processor
							input_feat_extract.pop("""reshaped_input_sizes"""						)  # pop reshaped_input_sizes as it is popped in the processor

							for key in input_feat_extract.keys():
									self.assertAlmostEqual(input_feat_extract[key].sum()      ,						input_processor[key].sum()      ,						delta=1e-2						)



					@require_tf
					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Dict		=			self.get_image_processor()

							SCREAMING_SNAKE_CASE :				Tuple		=			SamProcessor(image_processor=lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			[tf.ones((1, 3, 5, 5)						)]

							SCREAMING_SNAKE_CASE :				Dict		=			[[17_64, 26_46]]

							SCREAMING_SNAKE_CASE :				Any		=			[[6_83, 10_24]]
							SCREAMING_SNAKE_CASE :				str		=			processor.post_process_masks(lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_      ,						return_tensors="""tf"""						)
							self.assertEqual(masks[0].shape      ,						(1, 3, 17_64, 26_46)						)

							SCREAMING_SNAKE_CASE :				Optional[int]		=			processor.post_process_masks(
							    lowerCamelCase_      ,						tf.convert_to_tensor(lowerCamelCase_						)      ,						tf.convert_to_tensor(lowerCamelCase_						)      ,						return_tensors="""tf"""      ,						)
							self.assertEqual(masks[0].shape      ,						(1, 3, 17_64, 26_46)						)

							# should also work with np
							SCREAMING_SNAKE_CASE :				Optional[int]		=			[np.ones((1, 3, 5, 5)						)]
							SCREAMING_SNAKE_CASE :				List[str]		=			processor.post_process_masks(
							    lowerCamelCase_      ,						np.array(lowerCamelCase_						)      ,						np.array(lowerCamelCase_						)      ,						return_tensors="""tf"""						)

							self.assertEqual(masks[0].shape      ,						(1, 3, 17_64, 26_46)						)

							SCREAMING_SNAKE_CASE :				str		=			[[1, 0], [0, 1]]
							with self.assertRaises(tf.errors.InvalidArgumentError						):
									SCREAMING_SNAKE_CASE :				Tuple		=			processor.post_process_masks(
									    lowerCamelCase_      ,						np.array(lowerCamelCase_						)      ,						np.array(lowerCamelCase_						)      ,						return_tensors="""tf"""						)







@require_vision
@require_torchvision
class      UpperCamelCase__					(					unittest.TestCase			):




					"""simple docstring"""


					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				int		=			tempfile.mkdtemp()
							SCREAMING_SNAKE_CASE :				int		=			SamImageProcessor()
							SCREAMING_SNAKE_CASE :				str		=			SamProcessor(lowerCamelCase_						)
							processor.save_pretrained(self.tmpdirname						)


					def lowerCamelCase_			(						self				:			Tuple      ,						**lowerCamelCase_				:			List[Any]						):

							'''simple docstring'''



							return AutoProcessor.from_pretrained(self.tmpdirname      ,						**lowerCamelCase_						).image_processor


					def lowerCamelCase_			(						self				:			int						):

							'''simple docstring'''



							shutil.rmtree(self.tmpdirname						)


					def lowerCamelCase_			(						self				:			Dict						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Optional[int]		=			[np.random.randint(2_55      ,						size=(3, 30, 4_00)      ,						dtype=np.uinta						)]

							SCREAMING_SNAKE_CASE :				Dict		=			[Image.fromarray(np.moveaxis(lowerCamelCase_      ,						0      ,						-1						)						) for x in image_inputs]

							return image_inputs


					@is_pt_tf_cross_test
					def lowerCamelCase_			(						self				:			Optional[Any]						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				str		=			self.get_image_processor()

							SCREAMING_SNAKE_CASE :				Dict		=			SamProcessor(image_processor=lowerCamelCase_						)
							SCREAMING_SNAKE_CASE :				Any		=			np.random.randint(0      ,						2      ,						size=(1, 3, 5, 5)						).astype(np.floataa						)
							SCREAMING_SNAKE_CASE :				str		=			[tf.convert_to_tensor(lowerCamelCase_						)]
							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			[torch.tensor(lowerCamelCase_						)]

							SCREAMING_SNAKE_CASE :				Tuple		=			[[17_64, 26_46]]

							SCREAMING_SNAKE_CASE :				Optional[Any]		=			[[6_83, 10_24]]
							SCREAMING_SNAKE_CASE :				Tuple		=			processor.post_process_masks(
							    lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_      ,						return_tensors="""tf"""						)
							SCREAMING_SNAKE_CASE :				Any		=			processor.post_process_masks(
							    lowerCamelCase_      ,						lowerCamelCase_      ,						lowerCamelCase_      ,						return_tensors="""pt"""						)

							self.assertTrue(np.all(tf_masks[0].numpy() == pt_masks[0].numpy()						)						)



					@is_pt_tf_cross_test
					def lowerCamelCase_			(						self				:			Tuple						):

							'''simple docstring'''



							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			self.get_image_processor()

							SCREAMING_SNAKE_CASE :				Union[str, Any]		=			SamProcessor(image_processor=lowerCamelCase_						)

							SCREAMING_SNAKE_CASE :				Any		=			self.prepare_image_inputs()

							SCREAMING_SNAKE_CASE :				int		=			image_processor(lowerCamelCase_      ,						return_tensors="""pt"""						)["""pixel_values"""].numpy()
							SCREAMING_SNAKE_CASE :				Tuple		=			processor(images=lowerCamelCase_      ,						return_tensors="""pt"""						)["""pixel_values"""].numpy()

							SCREAMING_SNAKE_CASE :				Dict		=			image_processor(lowerCamelCase_      ,						return_tensors="""tf"""						)["""pixel_values"""].numpy()
							SCREAMING_SNAKE_CASE :				Dict		=			processor(images=lowerCamelCase_      ,						return_tensors="""tf"""						)["""pixel_values"""].numpy()

							self.assertTrue(np.allclose(lowerCamelCase_      ,						lowerCamelCase_						)						)
							self.assertTrue(np.allclose(lowerCamelCase_      ,						lowerCamelCase_						)						)
							self.assertTrue(np.allclose(lowerCamelCase_      ,						lowerCamelCase_						)						)



