


from typing import Dict, List, Optional, Union

import numpy as np

from transformers.utils import is_vision_available
from transformers.utils.generic import TensorType

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    center_crop,
    get_resize_output_image_size,
    normalize,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    is_valid_image,
    to_numpy_array,
    valid_images,
)
from ...utils import logging


if is_vision_available():
							import PIL

snake_case_						:			Union[str, Any]													=							logging.get_logger(__name__)







def 						lowerCamelCase(   a__):
				if isinstance(a__	,(list, tuple)) and isinstance(videos[0]	,(list, tuple)) and is_valid_image(videos[0][0]):
								return videos

				elif isinstance(a__	,(list, tuple)) and is_valid_image(videos[0]):
								return [videos]

				elif is_valid_image(a__):
								return [[videos]]

				raise ValueError(f"Could not make batched video from {videos}")


class 						A__							(		UpperCamelCase__  ):
						UpperCAmelCase             =						["pixel_values"]




						def __init__(       self					:      Tuple				,    _a					:      bool = True				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = PILImageResampling.BILINEAR				,    _a					:      bool = True				,    _a					:      Dict[str, int] = None				,    _a					:      bool = True				,    _a					:      Union[int, float] = 1 / 255				,    _a					:      bool = True				,    _a					:      bool = True				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    **_a					:      Any				,    )							->     None:


										"""simple docstring"""





										super().__init__(**_a  )
										_SCREAMING_SNAKE_CASE			  =size if size is not None else {'''shortest_edge''': 256}
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										_SCREAMING_SNAKE_CASE			  =crop_size if crop_size is not None else {'''height''': 224, '''width''': 224}
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    param_name='''crop_size'''  )

										_SCREAMING_SNAKE_CASE			  =do_resize
										_SCREAMING_SNAKE_CASE			  =size
										_SCREAMING_SNAKE_CASE			  =do_center_crop
										_SCREAMING_SNAKE_CASE			  =crop_size
										_SCREAMING_SNAKE_CASE			  =resample
										_SCREAMING_SNAKE_CASE			  =do_rescale
										_SCREAMING_SNAKE_CASE			  =rescale_factor
										_SCREAMING_SNAKE_CASE			  =offset
										_SCREAMING_SNAKE_CASE			  =do_normalize
										_SCREAMING_SNAKE_CASE			  =image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
										_SCREAMING_SNAKE_CASE			  =image_std if image_std is not None else IMAGENET_STANDARD_STD




						def 							__UpperCamelCase				(       self					:      List[Any]				,    _a					:      np.ndarray				,    _a					:      Dict[str, int]				,    _a					:      PILImageResampling = PILImageResampling.BILINEAR				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      List[Any]				,    )							->     np.ndarray:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										if "shortest_edge" in size:
														_SCREAMING_SNAKE_CASE			  =get_resize_output_image_size(_a				,    size['''shortest_edge''']				,    default_to_square=_a  )
										elif "height" in size and "width" in size:
														_SCREAMING_SNAKE_CASE			  =(size['''height'''], size['''width'''])
										else:
														raise ValueError(f"Size must have 'height' and 'width' or 'shortest_edge' as keys. Got {size.keys()}"  )
										return resize(_a				,    size=_a				,    resample=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      int				,    _a					:      np.ndarray				,    _a					:      Dict[str, int]				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      Dict				,    )							->     np.ndarray:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a  )
										if "height" not in size or "width" not in size:
														raise ValueError(f"Size must have 'height' and 'width' as keys. Got {size.keys()}"  )
										return center_crop(_a				,    size=(size['''height'''], size['''width'''])				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      np.ndarray				,    _a					:      Union[int, float]				,    _a					:      bool = True				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      List[str]				,    )							->     Tuple:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =image.astype(np.floataa  )
										if offset:
														_SCREAMING_SNAKE_CASE			  =image - (scale / 2)
										return rescale(_a				,    scale=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      List[str]				,    _a					:      np.ndarray				,    _a					:      Union[float, List[float]]				,    _a					:      Union[float, List[float]]				,    _a					:      Optional[Union[str, ChannelDimension]] = None				,    **_a					:      Any				,    )							->     np.ndarray:


										"""simple docstring"""





										return normalize(_a				,    mean=_a				,    std=_a				,    data_format=_a				,    **_a  )




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      ImageInput				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = None				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      bool = None				,    _a					:      float = None				,    _a					:      bool = None				,    _a					:      bool = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[ChannelDimension] = ChannelDimension.FIRST				,    )							->     np.ndarray:


										"""simple docstring"""





										if do_resize and size is None or resample is None:
														raise ValueError('''Size and resample must be specified if do_resize is True.'''  )

										if do_center_crop and crop_size is None:
														raise ValueError('''Crop size must be specified if do_center_crop is True.'''  )

										if do_rescale and rescale_factor is None:
														raise ValueError('''Rescale factor must be specified if do_rescale is True.'''  )

										if do_normalize and (image_mean is None or image_std is None):
														raise ValueError('''Image mean and std must be specified if do_normalize is True.'''  )

										if offset and not do_rescale:
														raise ValueError('''For offset, do_rescale must also be set to True.'''  )

										# All transformations expect numpy arrays.
										_SCREAMING_SNAKE_CASE			  =to_numpy_array(_a  )

										if do_resize:
														_SCREAMING_SNAKE_CASE			  =self.resize(image=_a				,    size=_a				,    resample=_a  )

										if do_center_crop:
														_SCREAMING_SNAKE_CASE			  =self.center_crop(_a				,    size=_a  )

										if do_rescale:
														_SCREAMING_SNAKE_CASE			  =self.rescale(image=_a				,    scale=_a				,    offset=_a  )

										if do_normalize:
														_SCREAMING_SNAKE_CASE			  =self.normalize(image=_a				,    mean=_a				,    std=_a  )

										_SCREAMING_SNAKE_CASE			  =to_channel_dimension_format(_a				,    _a  )
										return image




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      ImageInput				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      PILImageResampling = None				,    _a					:      bool = None				,    _a					:      Dict[str, int] = None				,    _a					:      bool = None				,    _a					:      float = None				,    _a					:      bool = None				,    _a					:      bool = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[float, List[float]]] = None				,    _a					:      Optional[Union[str, TensorType]] = None				,    _a					:      ChannelDimension = ChannelDimension.FIRST				,    **_a					:      str				,    )							->     PIL.Image.Image:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =do_resize if do_resize is not None else self.do_resize
										_SCREAMING_SNAKE_CASE			  =resample if resample is not None else self.resample
										_SCREAMING_SNAKE_CASE			  =do_center_crop if do_center_crop is not None else self.do_center_crop
										_SCREAMING_SNAKE_CASE			  =do_rescale if do_rescale is not None else self.do_rescale
										_SCREAMING_SNAKE_CASE			  =rescale_factor if rescale_factor is not None else self.rescale_factor
										_SCREAMING_SNAKE_CASE			  =offset if offset is not None else self.offset
										_SCREAMING_SNAKE_CASE			  =do_normalize if do_normalize is not None else self.do_normalize
										_SCREAMING_SNAKE_CASE			  =image_mean if image_mean is not None else self.image_mean
										_SCREAMING_SNAKE_CASE			  =image_std if image_std is not None else self.image_std

										_SCREAMING_SNAKE_CASE			  =size if size is not None else self.size
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    default_to_square=_a  )
										_SCREAMING_SNAKE_CASE			  =crop_size if crop_size is not None else self.crop_size
										_SCREAMING_SNAKE_CASE			  =get_size_dict(_a				,    param_name='''crop_size'''  )

										if not valid_images(_a  ):
														raise ValueError(
														    '''Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '''
														    '''torch.Tensor, tf.Tensor or jax.ndarray.'''  )

										_SCREAMING_SNAKE_CASE			  =make_batched(_a  )

										_SCREAMING_SNAKE_CASE			  =[
										    [
										        self._preprocess_image(
										            image=_a				,    do_resize=_a				,    size=_a				,    resample=_a				,    do_center_crop=_a				,    crop_size=_a				,    do_rescale=_a				,    rescale_factor=_a				,    offset=_a				,    do_normalize=_a				,    image_mean=_a				,    image_std=_a				,    data_format=_a				,    )
										        for img in video
										    ]
										    for video in videos
										]

										_SCREAMING_SNAKE_CASE			  ={'''pixel_values''': videos}
										return BatchFeature(data=_a				,    tensor_type=_a  )



import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


snake_case_						:			Any													=							datasets.utils.logging.get_logger(__name__)

snake_case_						:			List[Any]													=							['''names''', '''prefix''']
snake_case_						:			Dict													=							['''warn_bad_lines''', '''error_bad_lines''', '''mangle_dupe_cols''']
snake_case_						:			Dict													=							['''encoding_errors''', '''on_bad_lines''']
snake_case_						:			Optional[Any]													=							['''date_format''']

@dataclass
class 						A__							(		datasets.BuilderConfig  ):
						UpperCAmelCase             =						","
						UpperCAmelCase             =						None
						UpperCAmelCase             =						"infer"
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						True
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						False
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						True
						UpperCAmelCase             =						True
						UpperCAmelCase             =						False
						UpperCAmelCase             =						True
						UpperCAmelCase             =						None
						UpperCAmelCase             =						"."
						UpperCAmelCase             =						None
						UpperCAmelCase             =						'"'
						UpperCAmelCase             =						0
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						None
						UpperCAmelCase             =						True
						UpperCAmelCase             =						True
						UpperCAmelCase             =						0
						UpperCAmelCase             =						True
						UpperCAmelCase             =						False
						UpperCAmelCase             =						None
						UpperCAmelCase             =						10000
						UpperCAmelCase             =						None
						UpperCAmelCase             =						"strict"
						UpperCAmelCase             =						"error"
						UpperCAmelCase             =						None




						def 							__UpperCamelCase				(       self					:      int  )							->     Union[str, Any]:


										"""simple docstring"""





										if self.delimiter is not None:
														_SCREAMING_SNAKE_CASE			  =self.delimiter
										if self.column_names is not None:
														_SCREAMING_SNAKE_CASE			  =self.column_names




						@property
						def 							__UpperCamelCase				(       self					:      Dict  )							->     List[Any]:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  ={
										    '''sep''': self.sep,
										    '''header''': self.header,
										    '''names''': self.names,
										    '''index_col''': self.index_col,
										    '''usecols''': self.usecols,
										    '''prefix''': self.prefix,
										    '''mangle_dupe_cols''': self.mangle_dupe_cols,
										    '''engine''': self.engine,
										    '''converters''': self.converters,
										    '''true_values''': self.true_values,
										    '''false_values''': self.false_values,
										    '''skipinitialspace''': self.skipinitialspace,
										    '''skiprows''': self.skiprows,
										    '''nrows''': self.nrows,
										    '''na_values''': self.na_values,
										    '''keep_default_na''': self.keep_default_na,
										    '''na_filter''': self.na_filter,
										    '''verbose''': self.verbose,
										    '''skip_blank_lines''': self.skip_blank_lines,
										    '''thousands''': self.thousands,
										    '''decimal''': self.decimal,
										    '''lineterminator''': self.lineterminator,
										    '''quotechar''': self.quotechar,
										    '''quoting''': self.quoting,
										    '''escapechar''': self.escapechar,
										    '''comment''': self.comment,
										    '''encoding''': self.encoding,
										    '''dialect''': self.dialect,
										    '''error_bad_lines''': self.error_bad_lines,
										    '''warn_bad_lines''': self.warn_bad_lines,
										    '''skipfooter''': self.skipfooter,
										    '''doublequote''': self.doublequote,
										    '''memory_map''': self.memory_map,
										    '''float_precision''': self.float_precision,
										    '''chunksize''': self.chunksize,
										    '''encoding_errors''': self.encoding_errors,
										    '''on_bad_lines''': self.on_bad_lines,
										    '''date_format''': self.date_format,
										}

										# some kwargs must not be passed if they don't have a default value
										# some others are deprecated and we can also not pass them if they are the default value
										for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
														if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()				,    _a  ):
																		del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
										if not (datasets.config.PANDAS_VERSION.major >= 2):
														for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
																		del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
										if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
														for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
																		del pd_read_csv_kwargs[pd_read_csv_parameter]

										return pd_read_csv_kwargs

class 						A__							(		datasets.ArrowBasedBuilder  ):
						UpperCAmelCase             =						CsvConfig




						def 							__UpperCamelCase				(       self					:      Optional[int]  )							->     int:


										"""simple docstring"""





										return datasets.DatasetInfo(features=self.config.features  )




						def 							__UpperCamelCase				(       self					:      Dict				,    _a					:      str  )							->     List[str]:


										"""simple docstring"""





										if not self.config.data_files:
														raise ValueError(f"At least one data file must be specified, but got data_files={self.config.data_files}"  )
										_SCREAMING_SNAKE_CASE			  =dl_manager.download_and_extract(self.config.data_files  )
										if isinstance(_a				,    (str, list, tuple)  ):
														_SCREAMING_SNAKE_CASE			  =data_files
														if isinstance(_a				,    _a  ):
																		_SCREAMING_SNAKE_CASE			  =[files]
														_SCREAMING_SNAKE_CASE			  =[dl_manager.iter_files(_a  ) for file in files]
														return [datasets.SplitGenerator(name=datasets.Split.TRAIN				,    gen_kwargs={'''files''': files}  )]
										_SCREAMING_SNAKE_CASE			  =[]
										for split_name, files in data_files.items():
														if isinstance(_a				,    _a  ):
																		_SCREAMING_SNAKE_CASE			  =[files]
														_SCREAMING_SNAKE_CASE			  =[dl_manager.iter_files(_a  ) for file in files]
														splits.append(datasets.SplitGenerator(name=_a				,    gen_kwargs={'''files''': files}  )  )
										return splits




						def 							__UpperCamelCase				(       self					:      Tuple				,    _a					:      pa.Table  )							->     pa.Table:


										"""simple docstring"""





										if self.config.features is not None:
														_SCREAMING_SNAKE_CASE			  =self.config.features.arrow_schema
														if all(not require_storage_cast(_a  ) for feature in self.config.features.values()  ):
																		# cheaper cast
																		_SCREAMING_SNAKE_CASE			  =pa.Table.from_arrays([pa_table[field.name] for field in schema]				,    schema=_a  )
														else:
																		# more expensive cast; allows str <-> int/float or str to Audio for example
																		_SCREAMING_SNAKE_CASE			  =table_cast(_a				,    _a  )
										return pa_table




						def 							__UpperCamelCase				(       self					:      str				,    _a					:      Union[str, Any]  )							->     int:


										"""simple docstring"""





										_SCREAMING_SNAKE_CASE			  =self.config.features.arrow_schema if self.config.features else None
										# dtype allows reading an int column as str
										_SCREAMING_SNAKE_CASE			  =(
										    {
										        name: dtype.to_pandas_dtype() if not require_storage_cast(_a  ) else object
										        for name, dtype, feature in zip(schema.names				,    schema.types				,    self.config.features.values()  )
										    }
										    if schema is not None
										    else None
										)
										for file_idx, file in enumerate(itertools.chain.from_iterable(_a  )  ):
														_SCREAMING_SNAKE_CASE			  =pd.read_csv(_a				,    iterator=_a				,    dtype=_a				,    **self.config.pd_read_csv_kwargs  )
														try:
																		for batch_idx, df in enumerate(_a  ):
																						_SCREAMING_SNAKE_CASE			  =pa.Table.from_pandas(_a  )
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield (file_idx, batch_idx), self._cast_table(_a  )
														except ValueError as e:
																		logger.error(f"Failed to read file '{file}' with error {type(_a  )}: {e}"  )
																		raise