






"""simple docstring"""







import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


a :				int      					=		datasets.utils.logging.get_logger(__name__)


@dataclass
class     __UpperCAmelCase(   datasets.BuilderConfig							):




						"""simple docstring"""




						__lowerCamelCase				       =					None


class     __UpperCAmelCase(   datasets.ArrowBasedBuilder							):




						"""simple docstring"""




						__lowerCamelCase				       =					ArrowConfig




						def 		UpperCAmelCase_       (   self			):
												'''simple docstring'''
												return datasets.DatasetInfo(features=self.config.features			)




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												if not self.config.data_files:
																		raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''			)
												lowercase__				:      List[str]=       dl_manager.download_and_extract(self.config.data_files			)
												if isinstance(snake_case__       ,  (str, list, tuple)			):
																		lowercase__				:      Union[str, Any]=       data_files
																		if isinstance(snake_case__       ,  snake_case__			):
																								lowercase__				:      List[str]=       [files]
																		# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																		lowercase__				:      Optional[int]=       [dl_manager.iter_files(snake_case__			) for file in files]
																		return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,  gen_kwargs={"files": files}			)]
												lowercase__				:      Union[str, Any]=       []
												for split_name, files in data_files.items():
																		if isinstance(snake_case__       ,  snake_case__			):
																								lowercase__				:      Dict=       [files]
																		# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																		lowercase__				:      Optional[Any]=       [dl_manager.iter_files(snake_case__			) for file in files]
																		# Infer features is they are stoed in the arrow schema
																		if self.info.features is None:
																								for file in itertools.chain.from_iterable(snake_case__			):
																														with open(snake_case__       ,  "rb"			) as f:
																																				lowercase__				:      Optional[Any]=       datasets.Features.from_arrow_schema(pa.ipc.open_stream(snake_case__			).schema			)
																														break
																		splits.append(datasets.SplitGenerator(name=snake_case__       ,  gen_kwargs={"files": files}			)			)
												return splits




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												if self.info.features is not None:
																		# more expensive cast to support nested features with keys in a different order
																		# allows str <-> int/float or str to Audio for example
																		lowercase__				:      Any=       table_cast(snake_case__       ,  self.info.features.arrow_schema			)
												return pa_table




						def 		UpperCAmelCase_       (   self       ,  snake_case__			):
												'''simple docstring'''
												for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__			)			):
																		with open(snake_case__       ,  "rb"			) as f:
																								try:
																														for batch_idx, record_batch in enumerate(pa.ipc.open_stream(snake_case__			)			):
																																				lowercase__				:      Tuple=       pa.Table.from_batches([record_batch]			)
																																				# Uncomment for debugging (will print the Arrow table size and elements)
																																				# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																																				# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																																				yield F'''{file_idx}_{batch_idx}''', self._cast_table(snake_case__			)
																								except ValueError as e:
																														logger.error(F'''Failed to read file \'{file}\' with error {type(snake_case__			)}: {e}'''			)
																														raise











"""simple docstring"""
import unittest

import numpy as np

from transformers.testing_utils import require_torch, require_vision
from transformers.utils import is_torch_available, is_vision_available

from ...test_image_processing_common import ImageProcessingSavingTestMixin


if is_torch_available():
			import torch

if is_vision_available():
			from PIL import Image

			from transformers import BlipImageProcessor


class 							lowerCamelCase_(      unittest.TestCase			):





					'''simple docstring'''



					def __init__(			self ,				lowerCamelCase__ ,				lowerCamelCase__=7 ,				lowerCamelCase__=3 ,				lowerCamelCase__=1_8 ,				lowerCamelCase__=3_0 ,				lowerCamelCase__=4_0_0 ,				lowerCamelCase__=True ,				lowerCamelCase__=None ,				lowerCamelCase__=True ,				lowerCamelCase__=False ,				lowerCamelCase__=[0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3] ,				lowerCamelCase__=[0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1] ,				lowerCamelCase__=True ,				):
									_lowerCamelCase					     = size if size is not None else {'''height''': 2_0, '''width''': 2_0}
									_lowerCamelCase					     = parent
									_lowerCamelCase					     = batch_size
									_lowerCamelCase					     = num_channels
									_lowerCamelCase					     = image_size
									_lowerCamelCase					     = min_resolution
									_lowerCamelCase					     = max_resolution
									_lowerCamelCase					     = do_resize
									_lowerCamelCase					     = size
									_lowerCamelCase					     = do_normalize
									_lowerCamelCase					     = image_mean
									_lowerCamelCase					     = image_std
									_lowerCamelCase					     = do_pad
									_lowerCamelCase					     = do_convert_rgb

					def 	snake_case__					(			self      ):
									return {
									    "do_resize": self.do_resize,
									    "size": self.size,
									    "do_normalize": self.do_normalize,
									    "image_mean": self.image_mean,
									    "image_std": self.image_std,
									    "do_convert_rgb": self.do_convert_rgb,
									    "do_pad": self.do_pad,
									}





					def 	snake_case__					(			self ,				lowerCamelCase__=False ,				lowerCamelCase__=False ,				lowerCamelCase__=False      ):
									assert not (numpify and torchify), "You cannot specify both numpy and PyTorch tensors at the same time"

									if equal_resolution:
													_lowerCamelCase					     = []
													for i in range(self.batch_size      ):
																	image_inputs.append(
																	    np.random.randint(
																	        2_5_5 ,				size=(self.num_channels, self.max_resolution, self.max_resolution) ,				dtype=np.uinta      )      )
									else:
													_lowerCamelCase					     = []
													for i in range(self.batch_size      ):
																	_lowerCamelCase     ,   _lowerCamelCase					     = np.random.choice(np.arange(self.min_resolution ,				self.max_resolution      ) ,				2      )
																	image_inputs.append(np.random.randint(2_5_5 ,				size=(self.num_channels, width, height) ,				dtype=np.uinta      )      )

									if not numpify and not torchify:
													# PIL expects the channel dimension as last dimension
													_lowerCamelCase					     = [Image.fromarray(np.moveaxis(lowerCamelCase__ ,				0 ,				-1      )      ) for x in image_inputs]

									if torchify:
													_lowerCamelCase					     = [torch.from_numpy(lowerCamelCase__      ) for x in image_inputs]

									return image_inputs


@require_torch
@require_vision
class 							lowerCamelCase_(      A__, unittest.TestCase			):





					'''simple docstring'''



					lowercase__      :				Any        =       BlipImageProcessor if is_vision_available() else None

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = BlipImageProcessingTester(self      )

					@property
					def 	snake_case__					(			self      ):
									return self.image_processor_tester.prepare_image_processor_dict()

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.image_processing_class(**self.image_processor_dict      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''do_resize'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''size'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''do_normalize'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''image_mean'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''image_std'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''do_convert_rgb'''      )      )

					def 	snake_case__					(			self      ):
									pass

					def 	snake_case__					(			self      ):
									# Initialize image_processor
									_lowerCamelCase					     = self.image_processing_class(**self.image_processor_dict      )
									# create random PIL images
									_lowerCamelCase					     = self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase__      )
									for image in image_inputs:
													self.assertIsInstance(lowerCamelCase__ ,				Image.Image      )

									# Test not batched input
									_lowerCamelCase					     = image_processor(image_inputs[0] ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)

									# Test batched
									_lowerCamelCase					     = image_processor(lowerCamelCase__ ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)

					def 	snake_case__					(			self      ):
									# Initialize image_processor
									_lowerCamelCase					     = self.image_processing_class(**self.image_processor_dict      )
									# create random numpy tensors
									_lowerCamelCase					     = self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase__ ,				numpify=lowerCamelCase__      )
									for image in image_inputs:
													self.assertIsInstance(lowerCamelCase__ ,				np.ndarray      )

									# Test not batched input
									_lowerCamelCase					     = image_processor(image_inputs[0] ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)

									# Test batched
									_lowerCamelCase					     = image_processor(lowerCamelCase__ ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)





					def 	snake_case__					(			self      ):
									# Initialize image_processor
									_lowerCamelCase					     = self.image_processing_class(**self.image_processor_dict      )
									# create random PyTorch tensors
									_lowerCamelCase					     = self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase__ ,				torchify=lowerCamelCase__      )
									for image in image_inputs:
													self.assertIsInstance(lowerCamelCase__ ,				torch.Tensor      )

									# Test not batched input
									_lowerCamelCase					     = image_processor(image_inputs[0] ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        1,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)

									# Test batched
									_lowerCamelCase					     = image_processor(lowerCamelCase__ ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        self.image_processor_tester.batch_size,
									        self.image_processor_tester.num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)


@require_torch
@require_vision
class 							lowerCamelCase_(      A__, unittest.TestCase			):





					'''simple docstring'''



					lowercase__      :				Tuple        =       BlipImageProcessor if is_vision_available() else None

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = BlipImageProcessingTester(self ,				num_channels=4      )
									_lowerCamelCase					     = 3

					@property
					def 	snake_case__					(			self      ):
									return self.image_processor_tester.prepare_image_processor_dict()

					def 	snake_case__					(			self      ):
									_lowerCamelCase					     = self.image_processing_class(**self.image_processor_dict      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''do_resize'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''size'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''do_normalize'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''image_mean'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''image_std'''      )      )
									self.assertTrue(hasattr(lowerCamelCase__ ,				'''do_convert_rgb'''      )      )

					def 	snake_case__					(			self      ):
									pass





					def 	snake_case__					(			self      ):
									# Initialize image_processor
									_lowerCamelCase					     = self.image_processing_class(**self.image_processor_dict      )
									# create random PIL images
									_lowerCamelCase					     = self.image_processor_tester.prepare_inputs(equal_resolution=lowerCamelCase__      )
									for image in image_inputs:
													self.assertIsInstance(lowerCamelCase__ ,				Image.Image      )

									# Test not batched input
									_lowerCamelCase					     = image_processor(image_inputs[0] ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        1,
									        self.expected_encoded_image_num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)

									# Test batched
									_lowerCamelCase					     = image_processor(lowerCamelCase__ ,				return_tensors='''pt'''      ).pixel_values
									self.assertEqual(
									    encoded_images.shape ,				(
									        self.image_processor_tester.batch_size,
									        self.expected_encoded_image_num_channels,
									        self.image_processor_tester.size['''height'''],
									        self.image_processor_tester.size['''width'''],
									    ) ,				)
