






import json
import sys
import tempfile
import unittest
from pathlib import Path

import transformers
from transformers import (
    CONFIG_MAPPING,
    IMAGE_PROCESSOR_MAPPING,
    AutoConfig,
    AutoImageProcessor,
    CLIPConfig,
    CLIPImageProcessor,
)
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER


sys.path.append(str(Path(__file__).parent.parent.parent.parent / """utils"""))

from test_module.custom_configuration import CustomConfig  # noqa E402
from test_module.custom_image_processing import CustomImageProcessor  # noqa E402


class   __SCREAMING_SNAKE_CASE(	unittest.TestCase	):
							def lowerCAmelCase_ (     self:      Dict						)		->		List[str]:
											snake_case__									=     0
							def lowerCAmelCase_ (     self:      Optional[int]						)		->		str:
											snake_case__									=     AutoImageProcessor.from_pretrained('openai/clip-vit-base-patch32'						)
											self.assertIsInstance(UpperCamelCase       ,	UpperCamelCase						)
							def lowerCAmelCase_ (     self:      Dict						)		->		str:
											with tempfile.TemporaryDirectory() as tmpdirname:
															snake_case__									=     Path(UpperCamelCase						) / 'preprocessor_config.json'
															snake_case__									=     Path(UpperCamelCase						) / 'config.json'
															json.dump(
															    {'image_processor_type': 'CLIPImageProcessor', 'processor_class': 'CLIPProcessor'}       ,	open(UpperCamelCase       ,	'w'						)       ,	)
															json.dump({'model_type': 'clip'}       ,	open(UpperCamelCase       ,	'w'						)						)

															snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase						)
															self.assertIsInstance(UpperCamelCase       ,	UpperCamelCase						)
							def lowerCAmelCase_ (     self:      Dict						)		->		List[Any]:
											# Ensure we can load the image processor from the feature extractor config
											with tempfile.TemporaryDirectory() as tmpdirname:
															snake_case__									=     Path(UpperCamelCase						) / 'preprocessor_config.json'
															snake_case__									=     Path(UpperCamelCase						) / 'config.json'
															json.dump(
															    {'feature_extractor_type': 'CLIPFeatureExtractor', 'processor_class': 'CLIPProcessor'}       ,	open(UpperCamelCase       ,	'w'						)       ,	)
															json.dump({'model_type': 'clip'}       ,	open(UpperCamelCase       ,	'w'						)						)

															snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase						)
															self.assertIsInstance(UpperCamelCase       ,	UpperCamelCase						)
							def lowerCAmelCase_ (     self:      Dict						)		->		Optional[int]:
											with tempfile.TemporaryDirectory() as tmpdirname:
															snake_case__									=     CLIPConfig()

															# Create a dummy config file with image_proceesor_type
															snake_case__									=     Path(UpperCamelCase						) / 'preprocessor_config.json'
															snake_case__									=     Path(UpperCamelCase						) / 'config.json'
															json.dump(
															    {'image_processor_type': 'CLIPImageProcessor', 'processor_class': 'CLIPProcessor'}       ,	open(UpperCamelCase       ,	'w'						)       ,	)
															json.dump({'model_type': 'clip'}       ,	open(UpperCamelCase       ,	'w'						)						)

															# remove image_processor_type to make sure config.json alone is enough to load image processor locally
															snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase						).to_dict()

															config_dict.pop('image_processor_type'						)
															snake_case__									=     CLIPImageProcessor(**UpperCamelCase						)

															# save in new folder
															model_config.save_pretrained(UpperCamelCase						)
															config.save_pretrained(UpperCamelCase						)

															snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase						)

															# make sure private variable is not incorrectly saved
															snake_case__									=     json.loads(config.to_json_string()						)
															self.assertTrue('_processor_class' not in dict_as_saved						)

											self.assertIsInstance(UpperCamelCase       ,	UpperCamelCase						)
							def lowerCAmelCase_ (     self:      Optional[int]						)		->		Any:
											with tempfile.TemporaryDirectory() as tmpdirname:
															snake_case__									=     Path(UpperCamelCase						) / 'preprocessor_config.json'
															json.dump(
															    {'image_processor_type': 'CLIPImageProcessor', 'processor_class': 'CLIPProcessor'}       ,	open(UpperCamelCase       ,	'w'						)       ,	)

															snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase						)
															self.assertIsInstance(UpperCamelCase       ,	UpperCamelCase						)
							def lowerCAmelCase_ (     self:      Dict						)		->		Any:
											with self.assertRaisesRegex(
											    UpperCamelCase       ,	'clip-base is not a local folder and is not a valid model identifier'						):
															snake_case__									=     AutoImageProcessor.from_pretrained('clip-base'						)
							def lowerCAmelCase_ (     self:      List[Any]						)		->		str:
											with self.assertRaisesRegex(
											    UpperCamelCase       ,	R'aaaaaa is not a valid git identifier \(branch name, tag name or commit id\)'						):
															snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase       ,	revision='aaaaaa'						)
							def lowerCAmelCase_ (     self:      Union[str, Any]						)		->		Optional[int]:
											with self.assertRaisesRegex(
											    UpperCamelCase       ,	'hf-internal-testing/config-no-model does not appear to have a file named preprocessor_config.json.'       ,	):
															snake_case__									=     AutoImageProcessor.from_pretrained('hf-internal-testing/config-no-model'						)
							def lowerCAmelCase_ (     self:      List[Any]						)		->		str:
											# If remote code is not set, we will time out when asking whether to load the model.
											with self.assertRaises(UpperCamelCase						):
															snake_case__									=     AutoImageProcessor.from_pretrained('hf-internal-testing/test_dynamic_image_processor'						)
											# If remote code is disabled, we can't load this config.
											with self.assertRaises(UpperCamelCase						):
															snake_case__									=     AutoImageProcessor.from_pretrained(
															    'hf-internal-testing/test_dynamic_image_processor'       ,	trust_remote_code=UpperCamelCase						)

											snake_case__									=     AutoImageProcessor.from_pretrained(
											    'hf-internal-testing/test_dynamic_image_processor'       ,	trust_remote_code=UpperCamelCase						)
											self.assertEqual(image_processor.__class__.__name__       ,	'NewImageProcessor'						)

											# Test image processor can be reloaded.
											with tempfile.TemporaryDirectory() as tmp_dir:
															image_processor.save_pretrained(UpperCamelCase						)
															snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase       ,	trust_remote_code=UpperCamelCase						)
											self.assertEqual(reloaded_image_processor.__class__.__name__       ,	'NewImageProcessor'						)
							def lowerCAmelCase_ (     self:      Optional[Any]						)		->		Tuple:
											try:
															AutoConfig.register('custom'       ,	UpperCamelCase						)
															AutoImageProcessor.register(UpperCamelCase       ,	UpperCamelCase						)
															# Trying to register something existing in the Transformers library will raise an error
															with self.assertRaises(UpperCamelCase						):
																			AutoImageProcessor.register(UpperCamelCase       ,	UpperCamelCase						)

															with tempfile.TemporaryDirectory() as tmpdirname:
																			snake_case__									=     Path(UpperCamelCase						) / 'preprocessor_config.json'
																			snake_case__									=     Path(UpperCamelCase						) / 'config.json'
																			json.dump(
																			    {'feature_extractor_type': 'CLIPFeatureExtractor', 'processor_class': 'CLIPProcessor'}       ,	open(UpperCamelCase       ,	'w'						)       ,	)
																			json.dump({'model_type': 'clip'}       ,	open(UpperCamelCase       ,	'w'						)						)

																			snake_case__									=     CustomImageProcessor.from_pretrained(UpperCamelCase						)

															# Now that the config is registered, it can be used as any other config with the auto-API
															with tempfile.TemporaryDirectory() as tmp_dir:
																			image_processor.save_pretrained(UpperCamelCase						)
																			snake_case__									=     AutoImageProcessor.from_pretrained(UpperCamelCase						)
																			self.assertIsInstance(UpperCamelCase       ,	UpperCamelCase						)

											finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																			del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																			del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]


							def lowerCAmelCase_ (     self:      str						)		->		Tuple:


											class   __SCREAMING_SNAKE_CASE(	a_	):
																		_UpperCAmelCase     =       True

											try:
															AutoConfig.register('custom'       ,	UpperCamelCase						)
															AutoImageProcessor.register(UpperCamelCase       ,	UpperCamelCase						)
															# If remote code is not set, the default is to use local
															snake_case__									=     AutoImageProcessor.from_pretrained('hf-internal-testing/test_dynamic_image_processor'						)
															self.assertEqual(image_processor.__class__.__name__       ,	'NewImageProcessor'						)
															self.assertTrue(image_processor.is_local						)

															# If remote code is disabled, we load the local one.
															snake_case__									=     AutoImageProcessor.from_pretrained(
															    'hf-internal-testing/test_dynamic_image_processor'       ,	trust_remote_code=UpperCamelCase						)
															self.assertEqual(image_processor.__class__.__name__       ,	'NewImageProcessor'						)
															self.assertTrue(image_processor.is_local						)

															# If remote is enabled, we load from the Hub
															snake_case__									=     AutoImageProcessor.from_pretrained(
															    'hf-internal-testing/test_dynamic_image_processor'       ,	trust_remote_code=UpperCamelCase						)
															self.assertEqual(image_processor.__class__.__name__       ,	'NewImageProcessor'						)
															self.assertTrue(not hasattr(UpperCamelCase       ,	'is_local'						)						)

											finally:
															if "custom" in CONFIG_MAPPING._extra_content:
																			del CONFIG_MAPPING._extra_content["custom"]
															if CustomConfig in IMAGE_PROCESSOR_MAPPING._extra_content:
																			del IMAGE_PROCESSOR_MAPPING._extra_content[CustomConfig]












import itertools
from dataclasses import dataclass
from typing import Any, Callable, Dict, List, Optional, Union

import pandas as pd
import pyarrow as pa

import datasets
import datasets.config
from datasets.features.features import require_storage_cast
from datasets.table import table_cast
from datasets.utils.py_utils import Literal


__UpperCamelCase					:		Any	  =						datasets.utils.logging.get_logger(__name__)

__UpperCamelCase					:		Union[str, Any]	  =						["""names""", """prefix"""]
__UpperCamelCase					:		List[str]	  =						["""warn_bad_lines""", """error_bad_lines""", """mangle_dupe_cols"""]
__UpperCamelCase					:		Dict	  =						["""encoding_errors""", """on_bad_lines"""]
__UpperCamelCase					:		Any	  =						["""date_format"""]


@dataclass
class   __SCREAMING_SNAKE_CASE(	datasets.BuilderConfig	):
							_UpperCAmelCase     =       ","
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       "infer"
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       False
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       False
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       "."
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       '"'
							_UpperCAmelCase     =       0
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       0
							_UpperCAmelCase     =       True
							_UpperCAmelCase     =       False
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       1_0_0_0_0
							_UpperCAmelCase     =       None
							_UpperCAmelCase     =       "strict"
							_UpperCAmelCase     =       "error"
							_UpperCAmelCase     =       None
							def lowerCAmelCase_ (     self:      Tuple						)		->		int:
											if self.delimiter is not None:
															snake_case__									=     self.delimiter
											if self.column_names is not None:
															snake_case__									=     self.column_names


							@property
							def lowerCAmelCase_ (     self:      Optional[int]						)		->		List[str]:
											snake_case__									=     {
											    'sep': self.sep,
											    'header': self.header,
											    'names': self.names,
											    'index_col': self.index_col,
											    'usecols': self.usecols,
											    'prefix': self.prefix,
											    'mangle_dupe_cols': self.mangle_dupe_cols,
											    'engine': self.engine,
											    'converters': self.converters,
											    'true_values': self.true_values,
											    'false_values': self.false_values,
											    'skipinitialspace': self.skipinitialspace,
											    'skiprows': self.skiprows,
											    'nrows': self.nrows,
											    'na_values': self.na_values,
											    'keep_default_na': self.keep_default_na,
											    'na_filter': self.na_filter,
											    'verbose': self.verbose,
											    'skip_blank_lines': self.skip_blank_lines,
											    'thousands': self.thousands,
											    'decimal': self.decimal,
											    'lineterminator': self.lineterminator,
											    'quotechar': self.quotechar,
											    'quoting': self.quoting,
											    'escapechar': self.escapechar,
											    'comment': self.comment,
											    'encoding': self.encoding,
											    'dialect': self.dialect,
											    'error_bad_lines': self.error_bad_lines,
											    'warn_bad_lines': self.warn_bad_lines,
											    'skipfooter': self.skipfooter,
											    'doublequote': self.doublequote,
											    'memory_map': self.memory_map,
											    'float_precision': self.float_precision,
											    'chunksize': self.chunksize,
											    'encoding_errors': self.encoding_errors,
											    'on_bad_lines': self.on_bad_lines,
											    'date_format': self.date_format,
											}

											# some kwargs must not be passed if they don't have a default value
											# some others are deprecated and we can also not pass them if they are the default value
											for pd_read_csv_parameter in _PANDAS_READ_CSV_NO_DEFAULT_PARAMETERS + _PANDAS_READ_CSV_DEPRECATED_PARAMETERS:
															if pd_read_csv_kwargs[pd_read_csv_parameter] == getattr(CsvConfig()       ,	UpperCamelCase						):
																			del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 2.0 new arguments
											if not (datasets.config.PANDAS_VERSION.major >= 2):
															for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_2_0_0_PARAMETERS:
																			del pd_read_csv_kwargs[pd_read_csv_parameter]

        # Remove 1.3 new arguments
											if not (datasets.config.PANDAS_VERSION.major >= 1 and datasets.config.PANDAS_VERSION.minor >= 3):
															for pd_read_csv_parameter in _PANDAS_READ_CSV_NEW_1_3_0_PARAMETERS:
																			del pd_read_csv_kwargs[pd_read_csv_parameter]

											return pd_read_csv_kwargs


class   __SCREAMING_SNAKE_CASE(	datasets.ArrowBasedBuilder	):
							_UpperCAmelCase     =       CsvConfig
							def lowerCAmelCase_ (     self:      str						)		->		int:
											return datasets.DatasetInfo(features=self.config.features						)
							def lowerCAmelCase_ (     self:      Tuple       ,	UpperCamelCase:      Dict						)		->		int:
											if not self.config.data_files:
															raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''						)
											snake_case__									=     dl_manager.download_and_extract(self.config.data_files						)
											if isinstance(UpperCamelCase       ,	(str, list, tuple)						):
															snake_case__									=     data_files
															if isinstance(UpperCamelCase       ,	UpperCamelCase						):
																			snake_case__									=     [files]
															snake_case__									=     [dl_manager.iter_files(UpperCamelCase						) for file in files]
															return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,	gen_kwargs={'files': files}						)]
											snake_case__									=     []
											for split_name, files in data_files.items():
															if isinstance(UpperCamelCase       ,	UpperCamelCase						):
																			snake_case__									=     [files]
															snake_case__									=     [dl_manager.iter_files(UpperCamelCase						) for file in files]
															splits.append(datasets.SplitGenerator(name=UpperCamelCase       ,	gen_kwargs={'files': files}						)						)
											return splits
							def lowerCAmelCase_ (     self:      int       ,	UpperCamelCase:      pa.Table						)		->		pa.Table:
											if self.config.features is not None:
															snake_case__									=     self.config.features.arrow_schema
															if all(not require_storage_cast(UpperCamelCase						) for feature in self.config.features.values()						):
																			# cheaper cast
																			snake_case__									=     pa.Table.from_arrays([pa_table[field.name] for field in schema]       ,	schema=UpperCamelCase						)
															else:
																			# more expensive cast; allows str <-> int/float or str to Audio for example
																			snake_case__									=     table_cast(UpperCamelCase       ,	UpperCamelCase						)
											return pa_table


							def lowerCAmelCase_ (     self:      str       ,	UpperCamelCase:      Tuple						)		->		Tuple:
											snake_case__									=     self.config.features.arrow_schema if self.config.features else None
											# dtype allows reading an int column as str
											snake_case__									=     (
											    {
											        name: dtype.to_pandas_dtype() if not require_storage_cast(UpperCamelCase						) else object
											        for name, dtype, feature in zip(schema.names       ,	schema.types       ,	self.config.features.values()						)
											    }
											    if schema is not None
											    else None
											)
											for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase						)						):
															snake_case__									=     pd.read_csv(UpperCamelCase       ,	iterator=UpperCamelCase       ,	dtype=UpperCamelCase       ,	**self.config.pd_read_csv_kwargs						)
															try:
																			for batch_idx, df in enumerate(UpperCamelCase						):
																							snake_case__									=     pa.Table.from_pandas(UpperCamelCase						)
																							# Uncomment for debugging (will print the Arrow table size and elements)
																							# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																							# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																							yield (file_idx, batch_idx), self._cast_table(UpperCamelCase						)
															except ValueError as e:
																			logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase						)}: {e}'''						)
																			raise




