
import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


__a         =   datasets.utils.logging.get_logger(__name__)
@dataclass
class 					lowercase__(							datasets.BuilderConfig				):



	"""simple docstring"""




	a     :Optional[datasets.Features]       	=      None
	a     :str       	=      "utf-8"
	a     :Optional[str]       	=      None
	a     :Optional[str]       	=      None
	a     :bool       	=      True  # deprecated
	a     :Optional[int]       	=      None  # deprecated
	a     :int       	=      10 << 20  # 10MB
	a     :Optional[bool]       	=      None

class 					lowercase__(							datasets.ArrowBasedBuilder				):



	"""simple docstring"""




	a     :Any       	=      JsonConfig
	def 		_lowercase    (			self    :     Dict			)			->	Optional[int]:
					if self.config.block_size is not None:
									logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''			)
									lowercase_	    =   self.config.block_size
					if self.config.use_threads is not True:
									logger.warning(
									    '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''			)
					if self.config.newlines_in_values is not None:
									raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''			)
					return datasets.DatasetInfo(features=self.config.features			)
	def 		_lowercase    (			self    :     List[str]			,				SCREAMING_SNAKE_CASE_    :     Any			)			->	Union[str, Any]:
					if not self.config.data_files:
									raise ValueError(f'''At least one data file must be specified, but got data_files={self.config.data_files}'''			)
					lowercase_	    =   dl_manager.download_and_extract(self.config.data_files			)
					if isinstance(SCREAMING_SNAKE_CASE_			,				(str, list, tuple)			):
									lowercase_	    =   data_files
									if isinstance(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			):
													lowercase_	    =   [files]
									lowercase_	    =   [dl_manager.iter_files(SCREAMING_SNAKE_CASE_			) for file in files]
									return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,				gen_kwargs={'''files''': files}			)]
					lowercase_	    =   []
					for split_name, files in data_files.items():
									if isinstance(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			):
													lowercase_	    =   [files]
									lowercase_	    =   [dl_manager.iter_files(SCREAMING_SNAKE_CASE_			) for file in files]
									splits.append(datasets.SplitGenerator(name=SCREAMING_SNAKE_CASE_			,				gen_kwargs={'''files''': files}			)			)
					return splits
	def 		_lowercase    (			self    :     Any			,				SCREAMING_SNAKE_CASE_    :     pa.Table			)			->	pa.Table:
					if self.config.features is not None:
									# adding missing columns
									for column_name in set(self.config.features			) - set(pa_table.column_names			):
													lowercase_	    =   self.config.features.arrow_schema.field(SCREAMING_SNAKE_CASE_			).type
													lowercase_	    =   pa_table.append_column(SCREAMING_SNAKE_CASE_			,				pa.array([None] * len(SCREAMING_SNAKE_CASE_			)			,				type=SCREAMING_SNAKE_CASE_			)			)
									# more expensive cast to support nested structures with keys in a different order
									# allows str <-> int/float or str to Audio for example
									lowercase_	    =   table_cast(SCREAMING_SNAKE_CASE_			,				self.config.features.arrow_schema			)
					return pa_table




	def 		_lowercase    (			self    :     Dict			,				SCREAMING_SNAKE_CASE_    :     Tuple			)			->	List[str]:
					for file_idx, file in enumerate(itertools.chain.from_iterable(SCREAMING_SNAKE_CASE_			)			):
									# If the file is one json object and if we need to look at the list of items in one specific field
									if self.config.field is not None:
													with open(SCREAMING_SNAKE_CASE_			,				encoding=self.config.encoding			,				errors=self.config.encoding_errors			) as f:
																	lowercase_	    =   json.load(SCREAMING_SNAKE_CASE_			)

													# We keep only the field we are interested in
													lowercase_	    =   dataset[self.config.field]

													# We accept two format: a list of dicts or a dict of lists
													if isinstance(SCREAMING_SNAKE_CASE_			,				(list, tuple)			):
																	lowercase_	    =   set().union(*[row.keys() for row in dataset]			)
																	lowercase_	    =   {col: [row.get(SCREAMING_SNAKE_CASE_			) for row in dataset] for col in keys}
													else:
																	lowercase_	    =   dataset
													lowercase_	    =   pa.Table.from_pydict(SCREAMING_SNAKE_CASE_			)
													yield file_idx, self._cast_table(SCREAMING_SNAKE_CASE_			)

									# If the file has one json object per line
									else:
													with open(SCREAMING_SNAKE_CASE_			,				'''rb'''			) as f:
																	lowercase_	    =   0
																	# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																	# Set a default minimum value of 16kB if the chunk size is really small
																	lowercase_	    =   max(self.config.chunksize // 3_2			,				1_6 << 1_0			)
																	lowercase_	    =   (
																	    self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
																	)
																	while True:
																					lowercase_	    =   f.read(self.config.chunksize			)
																					if not batch:
																									break
																					# Finish current line
																					try:
																									batch += f.readline()
																					except (AttributeError, io.UnsupportedOperation):
																									batch += readline(SCREAMING_SNAKE_CASE_			)
																					# PyArrow only accepts utf-8 encoded bytes
																					if self.config.encoding != "utf-8":
																									lowercase_	    =   batch.decode(self.config.encoding			,				errors=SCREAMING_SNAKE_CASE_			).encode('''utf-8'''			)
																					try:
																									while True:
																													try:
																																	lowercase_	    =   paj.read_json(
																																	    io.BytesIO(SCREAMING_SNAKE_CASE_			)			,				read_options=paj.ReadOptions(block_size=SCREAMING_SNAKE_CASE_			)			)
																																	break
																													except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																	if (
																																	    isinstance(SCREAMING_SNAKE_CASE_			,				pa.ArrowInvalid			)
																																	    and "straddling" not in str(SCREAMING_SNAKE_CASE_			)
																																	    or block_size > len(SCREAMING_SNAKE_CASE_			)
																																	):
																																					raise
																																	else:
																																					# Increase the block size in case it was too small.
																																					# The block size will be reset for the next file.
																																					logger.debug(
																																					    f'''Batch of {len(SCREAMING_SNAKE_CASE_			)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''			)
																																					block_size *= 2
																					except pa.ArrowInvalid as e:
																									try:
																													with open(
																													    SCREAMING_SNAKE_CASE_			,				encoding=self.config.encoding			,				errors=self.config.encoding_errors			) as f:
																																	lowercase_	    =   json.load(SCREAMING_SNAKE_CASE_			)
																									except json.JSONDecodeError:
																													logger.error(f'''Failed to read file \'{file}\' with error {type(SCREAMING_SNAKE_CASE_			)}: {e}'''			)
																													raise e
																									# If possible, parse the file as a list of json objects and exit the loop
																									if isinstance(SCREAMING_SNAKE_CASE_			,				SCREAMING_SNAKE_CASE_			):  # list is the only sequence type supported in JSON
																													try:
																																	lowercase_	    =   set().union(*[row.keys() for row in dataset]			)
																																	lowercase_	    =   {col: [row.get(SCREAMING_SNAKE_CASE_			) for row in dataset] for col in keys}
																																	lowercase_	    =   pa.Table.from_pydict(SCREAMING_SNAKE_CASE_			)
																													except (pa.ArrowInvalid, AttributeError) as e:
																																	logger.error(f'''Failed to read file \'{file}\' with error {type(SCREAMING_SNAKE_CASE_			)}: {e}'''			)
																																	raise ValueError(f'''Not able to read records in the JSON file at {file}.'''			) from None
																													yield file_idx, self._cast_table(SCREAMING_SNAKE_CASE_			)
																													break
																									else:
																													logger.error(f'''Failed to read file \'{file}\' with error {type(SCREAMING_SNAKE_CASE_			)}: {e}'''			)
																													raise ValueError(
																													    f'''Not able to read records in the JSON file at {file}. '''
																													    f'''You should probably indicate the field of the JSON file containing your records. '''
																													    f'''This JSON file contain the following fields: {str(list(dataset.keys()			)			)}. '''
																													    f'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''			) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																					yield (file_idx, batch_idx), self._cast_table(SCREAMING_SNAKE_CASE_			)
																					batch_idx += 1






def 							a		(				snake_case__:   int   ):


				'''simple docstring'''
				if not isinstance(snake_case__		,  snake_case__   ):
								raise ValueError('''Input must be an integer'''   )
				if input_num <= 0:
								raise ValueError('''Input must be positive'''   )
				return sum(
				    divisor for divisor in range(1		,  input_num // 2 + 1   ) if input_num % divisor == 0   )


if __name__ == "__main__":
						import doctest

						doctest.testmod()




