




'''simple docstring'''


import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


_lowercase  =			datasets.utils.logging.get_logger(__name__)
@dataclass
class 						a_				(			datasets.BuilderConfig						):
	lowercase_       :     Optional[datasets.Features]			  =      None
	lowercase_       :     str			  =      "utf-8"
	lowercase_       :     Optional[str]			  =      None
	lowercase_       :     Optional[str]			  =      None
	lowercase_       :     bool			  =      True  # deprecated
	lowercase_       :     Optional[int]			  =      None  # deprecated
	lowercase_       :     int			  =      10 << 20  # 10MB
	lowercase_       :     Optional[bool]			  =      None





class 						a_				(			datasets.ArrowBasedBuilder						):
	lowercase_       :     int			  =      JsonConfig




	def   lowercase__   (       self     :  int      ):
							if self.config.block_size is not None:
													logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'      )
													__snake_case						 =   self.config.block_size
							if self.config.use_threads is not True:
													logger.warning(
													    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'      )
							if self.config.newlines_in_values is not None:
													raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'      )
							return datasets.DatasetInfo(features=self.config.features      )




	def   lowercase__   (       self     :  int       ,		__lowerCAmelCase     :  Dict      ):
							if not self.config.data_files:
													raise ValueError(F'At least one data file must be specified, but got data_files={self.config.data_files}'      )
							__snake_case						 =   dl_manager.download_and_extract(self.config.data_files      )
							if isinstance(__lowerCAmelCase       ,		(str, list, tuple)      ):
													__snake_case						 =   data_files
													if isinstance(__lowerCAmelCase       ,		__lowerCAmelCase      ):
																			__snake_case						 =   [files]
													__snake_case						 =   [dl_manager.iter_files(__lowerCAmelCase      ) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN       ,		gen_kwargs={'files': files}      )]
							__snake_case						 =   []
							for split_name, files in data_files.items():
													if isinstance(__lowerCAmelCase       ,		__lowerCAmelCase      ):
																			__snake_case						 =   [files]
													__snake_case						 =   [dl_manager.iter_files(__lowerCAmelCase      ) for file in files]
													splits.append(datasets.SplitGenerator(name=__lowerCAmelCase       ,		gen_kwargs={'files': files}      )      )
							return splits




	def   lowercase__   (       self     :  List[str]       ,		__lowerCAmelCase     :  pa.Table      ):
							if self.config.features is not None:
													# adding missing columns
													for column_name in set(self.config.features      ) - set(pa_table.column_names      ):
																			__snake_case						 =   self.config.features.arrow_schema.field(__lowerCAmelCase      ).type
																			__snake_case						 =   pa_table.append_column(__lowerCAmelCase       ,		pa.array([None] * len(__lowerCAmelCase      )       ,		type=__lowerCAmelCase      )      )
													# more expensive cast to support nested structures with keys in a different order
													# allows str <-> int/float or str to Audio for example
													__snake_case						 =   table_cast(__lowerCAmelCase       ,		self.config.features.arrow_schema      )
							return pa_table




	def   lowercase__   (       self     :  Tuple       ,		__lowerCAmelCase     :  Any      ):
							for file_idx, file in enumerate(itertools.chain.from_iterable(__lowerCAmelCase      )      ):
													# If the file is one json object and if we need to look at the list of items in one specific field
													if self.config.field is not None:
																			with open(__lowerCAmelCase       ,		encoding=self.config.encoding       ,		errors=self.config.encoding_errors      ) as f:
																									__snake_case						 =   json.load(__lowerCAmelCase      )

																			# We keep only the field we are interested in
																			__snake_case						 =   dataset[self.config.field]

																			# We accept two format: a list of dicts or a dict of lists
																			if isinstance(__lowerCAmelCase       ,		(list, tuple)      ):
																									__snake_case						 =   set().union(*[row.keys() for row in dataset]      )
																									__snake_case						 =   {col: [row.get(__lowerCAmelCase      ) for row in dataset] for col in keys}
																			else:
																									__snake_case						 =   dataset
																			__snake_case						 =   pa.Table.from_pydict(__lowerCAmelCase      )
																			yield file_idx, self._cast_table(__lowerCAmelCase      )

													# If the file has one json object per line
													else:
																			with open(__lowerCAmelCase       ,		'rb'      ) as f:
																									__snake_case						 =   0
																									# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																									# Set a default minimum value of 16kB if the chunk size is really small
																									__snake_case						 =   max(self.config.chunksize // 3_2       ,		1_6 << 1_0      )
																									__snake_case						 =   (
																									    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
																									)
																									while True:
																															__snake_case						 =   f.read(self.config.chunksize      )
																															if not batch:
																																					break
																															# Finish current line
																															try:
																																					batch += f.readline()
																															except (AttributeError, io.UnsupportedOperation):
																																					batch += readline(__lowerCAmelCase      )
																															# PyArrow only accepts utf-8 encoded bytes
																															if self.config.encoding != "utf-8":
																																					__snake_case						 =   batch.decode(self.config.encoding       ,		errors=__lowerCAmelCase      ).encode('utf-8'      )
																															try:
																																					while True:
																																											try:
																																																	__snake_case						 =   paj.read_json(
																																																	    io.BytesIO(__lowerCAmelCase      )       ,		read_options=paj.ReadOptions(block_size=__lowerCAmelCase      )      )
																																																	break
																																											except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																	if (
																																																	    isinstance(__lowerCAmelCase       ,		pa.ArrowInvalid      )
																																																	    and "straddling" not in str(__lowerCAmelCase      )
																																																	    or block_size > len(__lowerCAmelCase      )
																																																	):
																																																							raise
																																																	else:
																																																							# Increase the block size in case it was too small.
																																																							# The block size will be reset for the next file.
																																																							logger.debug(
																																																							    F'Batch of {len(__lowerCAmelCase      )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'      )
																																																							block_size *= 2
																															except pa.ArrowInvalid as e:
																																					try:
																																											with open(
																																											    __lowerCAmelCase       ,		encoding=self.config.encoding       ,		errors=self.config.encoding_errors      ) as f:
																																																	__snake_case						 =   json.load(__lowerCAmelCase      )
																																					except json.JSONDecodeError:
																																											logger.error(F'Failed to read file \'{file}\' with error {type(__lowerCAmelCase      )}: {e}'      )
																																											raise e
																																					# If possible, parse the file as a list of json objects and exit the loop
																																					if isinstance(__lowerCAmelCase       ,		__lowerCAmelCase      ):  # list is the only sequence type supported in JSON
																																											try:
																																																	__snake_case						 =   set().union(*[row.keys() for row in dataset]      )
																																																	__snake_case						 =   {col: [row.get(__lowerCAmelCase      ) for row in dataset] for col in keys}
																																																	__snake_case						 =   pa.Table.from_pydict(__lowerCAmelCase      )
																																											except (pa.ArrowInvalid, AttributeError) as e:
																																																	logger.error(F'Failed to read file \'{file}\' with error {type(__lowerCAmelCase      )}: {e}'      )
																																																	raise ValueError(F'Not able to read records in the JSON file at {file}.'      ) from None
																																											yield file_idx, self._cast_table(__lowerCAmelCase      )
																																											break
																																					else:
																																											logger.error(F'Failed to read file \'{file}\' with error {type(__lowerCAmelCase      )}: {e}'      )
																																											raise ValueError(
																																											    F'Not able to read records in the JSON file at {file}. '
																																											    F'You should probably indicate the field of the JSON file containing your records. '
																																											    F'This JSON file contain the following fields: {str(list(dataset.keys()      )      )}. '
																																											    F'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '      ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield (file_idx, batch_idx), self._cast_table(__lowerCAmelCase      )
																															batch_idx += 1







'''simple docstring'''


def 						lowerCamelCase__   (				a							):
						assert (
						    isinstance(a    ,	a							) and number_of_steps > 0
						), f'number_of_steps needs to be positive integer, your input {number_of_steps}'
						if number_of_steps == 1:
												return 1
						__snake_case ,       __snake_case						 =   1, 1
						for _ in range(number_of_steps - 1							):
												__snake_case ,       __snake_case						 =   current + previous, current
						return current


if __name__ == "__main__":
	import doctest

	doctest.testmod()

