





import numpy as np
import torch
from torch.utils.data import Dataset

from utils import logger




class 	UpperCamelCase_							(  UpperCamelCase__     ):


							def __init__(				self     :Any				,					__A     :Any				,					__A     :Any		) -> str:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   params

									SCREAMING_SNAKE_CASE__								=   np.array(__A		)
									SCREAMING_SNAKE_CASE__								=   np.array([len(__A		) for t in data]		)

									self.check()
									self.remove_long_sequences()
									self.remove_empty_sequences()
									self.remove_unknown_sequences()
									self.check()
									self.print_statistics()


							def __getitem__(				self     :Dict				,					__A     :Optional[Any]		) -> Optional[Any]:



									"""simple docstring"""



									return (self.token_ids[index], self.lengths[index])


							def __len__(				self     :str		) -> List[Any]:



									"""simple docstring"""



									return len(self.lengths		)


							def   _snake_case	(				self     :Tuple		) -> str:



									"""simple docstring"""



									assert len(self.token_ids		) == len(self.lengths		)
									assert all(self.lengths[i] == len(self.token_ids[i]		) for i in range(len(self.lengths		)		)		)


							def   _snake_case	(				self     :List[str]		) -> Tuple:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   self.params.max_model_input_size
									SCREAMING_SNAKE_CASE__								=   self.lengths > max_len
									logger.info(f'''Splitting {sum(__A		)} too long sequences.'''		)

									def divide_chunks(__A     :Dict				,					__A     :Optional[Any]		):
											return [l[i : i + n] for i in range(0				,					len(__A		)				,					__A		)]

									SCREAMING_SNAKE_CASE__								=   []
									SCREAMING_SNAKE_CASE__								=   []
									if self.params.mlm:
											SCREAMING_SNAKE_CASE__						,    SCREAMING_SNAKE_CASE__								=   self.params.special_tok_ids["""cls_token"""], self.params.special_tok_ids["""sep_token"""]
									else:
											SCREAMING_SNAKE_CASE__						,    SCREAMING_SNAKE_CASE__								=   self.params.special_tok_ids["""bos_token"""], self.params.special_tok_ids["""eos_token"""]

									for seq_, len_ in zip(self.token_ids				,					self.lengths		):
											assert (seq_[0] == cls_id) and (seq_[-1] == sep_id), seq_
											if len_ <= max_len:
													new_tok_ids.append(seq_		)
													new_lengths.append(len_		)
											else:
													SCREAMING_SNAKE_CASE__								=   []
													for sub_s in divide_chunks(seq_				,					max_len - 2		):
															if sub_s[0] != cls_id:
																	SCREAMING_SNAKE_CASE__								=   np.insert(__A				,					0				,					__A		)
															if sub_s[-1] != sep_id:
																	SCREAMING_SNAKE_CASE__								=   np.insert(__A				,					len(__A		)				,					__A		)
															assert len(__A		) <= max_len
															assert (sub_s[0] == cls_id) and (sub_s[-1] == sep_id), sub_s
															sub_seqs.append(__A		)

													new_tok_ids.extend(__A		)
													new_lengths.extend([len(__A		) for l in sub_seqs]		)

									SCREAMING_SNAKE_CASE__								=   np.array(__A		)
									SCREAMING_SNAKE_CASE__								=   np.array(__A		)


							def   _snake_case	(				self     :str		) -> Optional[Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   len(self		)
									SCREAMING_SNAKE_CASE__								=   self.lengths > 11
									SCREAMING_SNAKE_CASE__								=   self.token_ids[indices]
									SCREAMING_SNAKE_CASE__								=   self.lengths[indices]
									SCREAMING_SNAKE_CASE__								=   len(self		)
									logger.info(f'''Remove {init_size - new_size} too short (<=11 tokens) sequences.'''		)


							def   _snake_case	(				self     :str		) -> Any:



									"""simple docstring"""



									if "unk_token" not in self.params.special_tok_ids:
											return
									else:
											SCREAMING_SNAKE_CASE__								=   self.params.special_tok_ids["""unk_token"""]
									SCREAMING_SNAKE_CASE__								=   len(self		)
									SCREAMING_SNAKE_CASE__								=   np.array([np.count_nonzero(a == unk_token_id		) for a in self.token_ids]		)
									SCREAMING_SNAKE_CASE__								=   (unk_occs / self.lengths) < 0.5
									SCREAMING_SNAKE_CASE__								=   self.token_ids[indices]
									SCREAMING_SNAKE_CASE__								=   self.lengths[indices]
									SCREAMING_SNAKE_CASE__								=   len(self		)
									logger.info(f'''Remove {init_size - new_size} sequences with a high level of unknown tokens (50%).'''		)


							def   _snake_case	(				self     :Union[str, Any]		) -> List[str]:



									"""simple docstring"""



									if not self.params.is_master:
											return
									logger.info(f'''{len(self		)} sequences'''		)
									# data_len = sum(self.lengths)
									# nb_unique_tokens = len(Counter(list(chain(*self.token_ids))))
									# logger.info(f'{data_len} tokens ({nb_unique_tokens} unique)')

									# unk_idx = self.params.special_tok_ids['unk_token']
									# nb_unknown = sum([(t==unk_idx).sum() for t in self.token_ids])
									# logger.info(f'{nb_unknown} unknown tokens (covering {100*nb_unknown/data_len:.2f}% of the data)')


							def   _snake_case	(				self     :List[Any]				,					__A     :Dict		) -> Union[str, Any]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   [t[0] for t in batch]
									SCREAMING_SNAKE_CASE__								=   [t[1] for t in batch]
									assert len(__A		) == len(__A		)

									# Max for paddings
									SCREAMING_SNAKE_CASE__								=   max(__A		)

									# Pad token ids
									if self.params.mlm:
											SCREAMING_SNAKE_CASE__								=   self.params.special_tok_ids["""pad_token"""]
									else:
											SCREAMING_SNAKE_CASE__								=   self.params.special_tok_ids["""unk_token"""]
									SCREAMING_SNAKE_CASE__								=   [list(t.astype(__A		)		) + [pad_idx] * (max_seq_len_ - len(__A		)) for t in token_ids]
									assert len(tk_		) == len(__A		)
									assert all(len(__A		) == max_seq_len_ for t in tk_		)

									SCREAMING_SNAKE_CASE__								=   torch.tensor(tk_		)  # (bs, max_seq_len_)
									SCREAMING_SNAKE_CASE__								=   torch.tensor(__A		)  # (bs)
									return tk_t, lg_t






import warnings
from typing import List

import numpy as np

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
from ...utils import is_flax_available, is_tf_available, is_torch_available




class 	UpperCamelCase_							(  UpperCamelCase__     ):
							lowerCamelCase_							     =       ["image_processor", "tokenizer"]
							lowerCamelCase_							     =       "OwlViTImageProcessor"
							lowerCamelCase_							     =       ("CLIPTokenizer", "CLIPTokenizerFast")


							def __init__(				self     :Optional[Any]				,					__A     :int=None				,					__A     :Optional[int]=None				,					**__A     :str		) -> Optional[int]:



									"""simple docstring"""



									SCREAMING_SNAKE_CASE__								=   None
									if "feature_extractor" in kwargs:
											warnings.warn(
											    """The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`"""
											    """ instead."""				,					__A				,					)
											SCREAMING_SNAKE_CASE__								=   kwargs.pop("""feature_extractor"""		)

									SCREAMING_SNAKE_CASE__								=   image_processor if image_processor is not None else feature_extractor
									if image_processor is None:
											raise ValueError("""You need to specify an `image_processor`."""		)
									if tokenizer is None:
											raise ValueError("""You need to specify a `tokenizer`."""		)

									super().__init__(__A				,					__A		)


							def __call__(				self     :str				,					__A     :Dict=None				,					__A     :List[str]=None				,					__A     :str=None				,					__A     :Optional[int]="max_length"				,					__A     :Tuple="np"				,					**__A     :int		) -> Tuple:



									"""simple docstring"""



									if text is None and query_images is None and images is None:
											raise ValueError(
											    """You have to specify at least one text or query image or image. All three cannot be none."""		)

									if text is not None:
											if isinstance(__A				,					__A		) or (isinstance(__A				,					__A		) and not isinstance(text[0]				,					__A		)):
													SCREAMING_SNAKE_CASE__								=   [self.tokenizer(__A				,					padding=__A				,					return_tensors=__A				,					**__A		)]

											elif isinstance(__A				,					__A		) and isinstance(text[0]				,					__A		):
													SCREAMING_SNAKE_CASE__								=   []

													# Maximum number of queries across batch
													SCREAMING_SNAKE_CASE__								=   max([len(__A		) for t in text]		)

													# Pad all batch samples to max number of text queries
													for t in text:
															if len(__A		) != max_num_queries:
																	SCREAMING_SNAKE_CASE__								=   t + [""" """] * (max_num_queries - len(__A		))

															SCREAMING_SNAKE_CASE__								=   self.tokenizer(__A				,					padding=__A				,					return_tensors=__A				,					**__A		)
															encodings.append(__A		)
											else:
													raise TypeError("""Input text should be a string, a list of strings or a nested list of strings"""		)

											if return_tensors == "np":
													SCREAMING_SNAKE_CASE__								=   np.concatenate([encoding["""input_ids"""] for encoding in encodings]				,					axis=0		)
													SCREAMING_SNAKE_CASE__								=   np.concatenate([encoding["""attention_mask"""] for encoding in encodings]				,					axis=0		)

											elif return_tensors == "jax" and is_flax_available():
													import jax.numpy as jnp

													SCREAMING_SNAKE_CASE__								=   jnp.concatenate([encoding["""input_ids"""] for encoding in encodings]				,					axis=0		)
													SCREAMING_SNAKE_CASE__								=   jnp.concatenate([encoding["""attention_mask"""] for encoding in encodings]				,					axis=0		)

											elif return_tensors == "pt" and is_torch_available():
													import torch

													SCREAMING_SNAKE_CASE__								=   torch.cat([encoding["""input_ids"""] for encoding in encodings]				,					dim=0		)
													SCREAMING_SNAKE_CASE__								=   torch.cat([encoding["""attention_mask"""] for encoding in encodings]				,					dim=0		)

											elif return_tensors == "tf" and is_tf_available():
													import tensorflow as tf

													SCREAMING_SNAKE_CASE__								=   tf.stack([encoding["""input_ids"""] for encoding in encodings]				,					axis=0		)
													SCREAMING_SNAKE_CASE__								=   tf.stack([encoding["""attention_mask"""] for encoding in encodings]				,					axis=0		)

											else:
													raise ValueError("""Target return tensor type could not be returned"""		)

											SCREAMING_SNAKE_CASE__								=   BatchEncoding()
											SCREAMING_SNAKE_CASE__								=   input_ids
											SCREAMING_SNAKE_CASE__								=   attention_mask

									if query_images is not None:
											SCREAMING_SNAKE_CASE__								=   BatchEncoding()
											SCREAMING_SNAKE_CASE__								=   self.image_processor(
											    __A				,					return_tensors=__A				,					**__A		).pixel_values
											SCREAMING_SNAKE_CASE__								=   query_pixel_values

									if images is not None:
											SCREAMING_SNAKE_CASE__								=   self.image_processor(__A				,					return_tensors=__A				,					**__A		)

									if text is not None and images is not None:
											SCREAMING_SNAKE_CASE__								=   image_features.pixel_values
											return encoding
									elif query_images is not None and images is not None:
											SCREAMING_SNAKE_CASE__								=   image_features.pixel_values
											return encoding
									elif text is not None or query_images is not None:
											return encoding
									else:
											return BatchEncoding(data=dict(**__A		)				,					tensor_type=__A		)


							def   _snake_case	(				self     :List[Any]				,					*__A     :Dict				,					**__A     :Dict		) -> Optional[int]:



									"""simple docstring"""



									return self.image_processor.post_process(*__A				,					**__A		)


							def   _snake_case	(				self     :Optional[int]				,					*__A     :Dict				,					**__A     :List[str]		) -> Optional[Any]:



									"""simple docstring"""



									return self.image_processor.post_process_object_detection(*__A				,					**__A		)


							def   _snake_case	(				self     :str				,					*__A     :List[str]				,					**__A     :Union[str, Any]		) -> Any:



									"""simple docstring"""



									return self.image_processor.post_process_image_guided_detection(*__A				,					**__A		)


							def   _snake_case	(				self     :Dict				,					*__A     :List[str]				,					**__A     :List[str]		) -> int:



									"""simple docstring"""



									return self.tokenizer.batch_decode(*__A				,					**__A		)


							def   _snake_case	(				self     :Dict				,					*__A     :Dict				,					**__A     :List[str]		) -> str:



									"""simple docstring"""



									return self.tokenizer.decode(*__A				,					**__A		)


							@property
							def   _snake_case	(				self     :List[Any]		) -> Optional[int]:



									"""simple docstring"""



									warnings.warn(
									    """`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead."""				,					__A				,					)
									return self.image_processor_class


							@property
							def   _snake_case	(				self     :Any		) -> Optional[Any]:



									"""simple docstring"""



									warnings.warn(
									    """`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead."""				,					__A				,					)
									return self.image_processor