


"""simple docstring"""






import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__snake_case		=       datasets.utils.logging.get_logger(__name__)


@dataclass
class     __lowerCamelCase  (					datasets.BuilderConfig				):

     '''simple docstring'''
     A_       :	int    =		10_000
     A_       :	Optional[List[str]]    =		None
     A_       :	Optional[datasets.Features]    =		None







class     __lowerCamelCase  (					datasets.ArrowBasedBuilder				):

     '''simple docstring'''
     A_       :	Any    =		ParquetConfig


     def 							_UpperCAmelCase      (      self							)       ->							List[Any]:
          return datasets.DatasetInfo(features=self.config.features							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							Tuple:
          if not self.config.data_files:
               raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'							)
          _a    	=				dl_manager.download_and_extract(self.config.data_files							)
          if isinstance(__UpperCAmelCase    ,				(str, list, tuple)							):
               _a    	=				data_files
               if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							):
                    _a    	=				[files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               _a    	=				[dl_manager.iter_files(__UpperCAmelCase							) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,				gen_kwargs={'''files''': files}							)]
          _a    	=				[]
          for split_name, files in data_files.items():
               if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							):
                    _a    	=				[files]
               # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
               _a    	=				[dl_manager.iter_files(__UpperCAmelCase							) for file in files]
               # Infer features is they are stoed in the arrow schema
               if self.info.features is None:
                    for file in itertools.chain.from_iterable(__UpperCAmelCase							):
                         with open(__UpperCAmelCase    ,				'''rb'''							) as f:
                              _a    	=				datasets.Features.from_arrow_schema(pq.read_schema(__UpperCAmelCase							)							)
                         break
               splits.append(datasets.SplitGenerator(name=__UpperCAmelCase    ,				gen_kwargs={'''files''': files}							)							)
          return splits


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							pa.Table:
          if self.info.features is not None:
               # more expensive cast to support nested features with keys in a different order
               # allows str <-> int/float or str to Audio for example
               _a    	=				table_cast(__UpperCAmelCase    ,				self.info.features.arrow_schema							)
          return pa_table





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase							)       ->							str:
          _a    	=				self.info.features.arrow_schema if self.info.features is not None else None
          if self.info.features is not None and self.config.columns is not None:
               if sorted(field.name for field in schema							) != sorted(self.config.columns							):
                    raise ValueError(
                        F'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''							)
          for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCAmelCase							)							):
               with open(__UpperCAmelCase    ,				'''rb'''							) as f:
                    _a    	=				pq.ParquetFile(__UpperCAmelCase							)
                    try:
                         for batch_idx, record_batch in enumerate(
                             parquet_file.iter_batches(batch_size=self.config.batch_size    ,				columns=self.config.columns							)							):
                              _a    	=				pa.Table.from_batches([record_batch]							)
                              # Uncomment for debugging (will print the Arrow table size and elements)
                              # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                              # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                              yield F'{file_idx}_{batch_idx}', self._cast_table(__UpperCAmelCase							)
                    except ValueError as e:
                         logger.error(F'Failed to read file \'{file}\' with error {type(__UpperCAmelCase							)}: {e}'							)
                         raise



"""simple docstring"""






from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, is_vision_available, logging


if is_vision_available():
   import PIL


__snake_case		=       logging.get_logger(__name__)


class     __lowerCamelCase  (					a__				):

     '''simple docstring'''
     A_       :	Any    =		['pixel_values']


     def __init__(      self    ,				__UpperCAmelCase = True    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = PILImageResampling.BILINEAR    ,				__UpperCAmelCase = True    ,				__UpperCAmelCase = 1 / 255    ,				__UpperCAmelCase = True    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							None:
          super().__init__(**__UpperCAmelCase							)
          _a    	=				size if size is not None else {'''shortest_edge''': 384}
          _a    	=				get_size_dict(__UpperCAmelCase    ,				default_to_square=__UpperCAmelCase							)

          _a    	=				do_resize
          _a    	=				size
          # Default value set here for backwards compatibility where the value in config is None
          _a    	=				crop_pct if crop_pct is not None else 224 / 256
          _a    	=				resample
          _a    	=				do_rescale
          _a    	=				rescale_factor
          _a    	=				do_normalize
          _a    	=				image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
          _a    	=				image_std if image_std is not None else IMAGENET_STANDARD_STD


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase = PILImageResampling.BICUBIC    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							np.ndarray:
          _a    	=				get_size_dict(__UpperCAmelCase    ,				default_to_square=__UpperCAmelCase							)
          if "shortest_edge" not in size:
               raise ValueError(F'Size dictionary must contain \'shortest_edge\' key. Got {size.keys()}'							)
          _a    	=				size['''shortest_edge''']

          if shortest_edge < 384:
               # maintain same ratio, resizing shortest edge to shortest_edge/crop_pct
               _a    	=				int(shortest_edge / crop_pct							)
               _a    	=				get_resize_output_image_size(__UpperCAmelCase    ,				size=__UpperCAmelCase    ,				default_to_square=__UpperCAmelCase							)
               _a    	=				resize(image=__UpperCAmelCase    ,				size=__UpperCAmelCase    ,				resample=__UpperCAmelCase    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)
               # then crop to (shortest_edge, shortest_edge)
               return center_crop(image=__UpperCAmelCase    ,				size=(shortest_edge, shortest_edge)    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)
          else:
               # warping (no cropping) when evaluated at 384 or larger
               return resize(
                   __UpperCAmelCase    ,				size=(shortest_edge, shortest_edge)    ,				resample=__UpperCAmelCase    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							Optional[int]:
          return rescale(__UpperCAmelCase    ,				scale=__UpperCAmelCase    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				**__UpperCAmelCase    ,				)       ->							np.ndarray:
          return normalize(__UpperCAmelCase    ,				mean=__UpperCAmelCase    ,				std=__UpperCAmelCase    ,				data_format=__UpperCAmelCase    ,				**__UpperCAmelCase							)





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = None    ,				__UpperCAmelCase = ChannelDimension.FIRST    ,				**__UpperCAmelCase    ,				)       ->							PIL.Image.Image:
          _a    	=				do_resize if do_resize is not None else self.do_resize
          _a    	=				crop_pct if crop_pct is not None else self.crop_pct
          _a    	=				resample if resample is not None else self.resample
          _a    	=				do_rescale if do_rescale is not None else self.do_rescale
          _a    	=				rescale_factor if rescale_factor is not None else self.rescale_factor
          _a    	=				do_normalize if do_normalize is not None else self.do_normalize
          _a    	=				image_mean if image_mean is not None else self.image_mean
          _a    	=				image_std if image_std is not None else self.image_std

          _a    	=				size if size is not None else self.size
          _a    	=				get_size_dict(__UpperCAmelCase    ,				default_to_square=__UpperCAmelCase							)

          _a    	=				make_list_of_images(__UpperCAmelCase							)

          if not valid_images(__UpperCAmelCase							):
               raise ValueError(
                   '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
                   '''torch.Tensor, tf.Tensor or jax.ndarray.'''							)

          if do_resize and size is None or resample is None:
               raise ValueError('''Size and resample must be specified if do_resize is True.'''							)

          if do_resize and size["shortest_edge"] < 384 and crop_pct is None:
               raise ValueError('''crop_pct must be specified if size < 384.'''							)

          if do_rescale and rescale_factor is None:
               raise ValueError('''Rescale factor must be specified if do_rescale is True.'''							)

          if do_normalize and (image_mean is None or image_std is None):
               raise ValueError('''Image mean and std must be specified if do_normalize is True.'''							)

          # All transformations expect numpy arrays.
          _a    	=				[to_numpy_array(__UpperCAmelCase							) for image in images]

          if do_resize:
               _a    	=				[self.resize(image=__UpperCAmelCase    ,				size=__UpperCAmelCase    ,				crop_pct=__UpperCAmelCase    ,				resample=__UpperCAmelCase							) for image in images]

          if do_rescale:
               _a    	=				[self.rescale(image=__UpperCAmelCase    ,				scale=__UpperCAmelCase							) for image in images]

          if do_normalize:
               _a    	=				[self.normalize(image=__UpperCAmelCase    ,				mean=__UpperCAmelCase    ,				std=__UpperCAmelCase							) for image in images]

          _a    	=				[to_channel_dimension_format(__UpperCAmelCase    ,				__UpperCAmelCase							) for image in images]

          _a    	=				{'''pixel_values''': images}
          return BatchFeature(data=__UpperCAmelCase    ,				tensor_type=__UpperCAmelCase							)