"""simple docstring"""







import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


SCREAMING_SNAKE_CASE									=						datasets.utils.logging.get_logger(__name__)



@dataclass
class  _lowerCamelCase			(datasets.BuilderConfig     ):
				_snake_case     =  None
				_snake_case     =  "utf-8"
				_snake_case     =  None
				_snake_case     =  None
				_snake_case     =  True  # deprecated
				_snake_case     =  None  # deprecated
				_snake_case     =  10 << 20  # 10MB
				_snake_case     =  None



class  _lowerCamelCase			(datasets.ArrowBasedBuilder     ):
				_snake_case     =  JsonConfig







				def 							__UpperCAmelCase					( self    :       Union[str, Any]						):

					"""simple docstring"""






					if self.config.block_size is not None:
						logger.warning('The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'						)
						_lowercase :    str							=							self.config.block_size
					if self.config.use_threads is not True:
						logger.warning(
						    'The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'						)
					if self.config.newlines_in_values is not None:
						raise ValueError('The JSON loader parameter `newlines_in_values` is no longer supported'						)
					return datasets.DatasetInfo(features=self.config.features						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]			,					lowerCamelCase_    :       List[Any]						):

					"""simple docstring"""






					if not self.config.data_files:
						raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''						)
					_lowercase :    Optional[int]							=							dl_manager.download_and_extract(self.config.data_files						)
					if isinstance(lowerCamelCase_			,					(str, list, tuple)						):
						_lowercase :    Optional[Any]							=							data_files
						if isinstance(lowerCamelCase_			,					lowerCamelCase_						):
							_lowercase :    Any							=							[files]
						_lowercase :    Optional[int]							=							[dl_manager.iter_files(lowerCamelCase_						) for file in files]
						return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,					gen_kwargs={'files': files}						)]
					_lowercase :    List[str]							=							[]
					for split_name, files in data_files.items():
						if isinstance(lowerCamelCase_			,					lowerCamelCase_						):
							_lowercase :    List[str]							=							[files]
						_lowercase :    Dict							=							[dl_manager.iter_files(lowerCamelCase_						) for file in files]
						splits.append(datasets.SplitGenerator(name=lowerCamelCase_			,					gen_kwargs={'files': files}						)						)
					return splits







				def 							__UpperCAmelCase					( self    :       str			,					lowerCamelCase_    :       pa.Table						):

					"""simple docstring"""






					if self.config.features is not None:
						# adding missing columns
						for column_name in set(self.config.features						) - set(pa_table.column_names						):
							_lowercase :    Union[str, Any]							=							self.config.features.arrow_schema.field(lowerCamelCase_						).type
							_lowercase :    List[str]							=							pa_table.append_column(lowerCamelCase_			,					pa.array([None] * len(lowerCamelCase_						)			,					type=lowerCamelCase_						)						)
						# more expensive cast to support nested structures with keys in a different order
						# allows str <-> int/float or str to Audio for example
						_lowercase :    Optional[Any]							=							table_cast(lowerCamelCase_			,					self.config.features.arrow_schema						)
					return pa_table







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       Optional[int]						):

					"""simple docstring"""






					for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase_						)						):
						# If the file is one json object and if we need to look at the list of items in one specific field
						if self.config.field is not None:
							with open(lowerCamelCase_			,					encoding=self.config.encoding			,					errors=self.config.encoding_errors						) as f:
								_lowercase :    Union[str, Any]							=							json.load(lowerCamelCase_						)

							# We keep only the field we are interested in
							_lowercase :    Tuple							=							dataset[self.config.field]

							# We accept two format: a list of dicts or a dict of lists
							if isinstance(lowerCamelCase_			,					(list, tuple)						):
								_lowercase :    List[str]							=							set().union(*[row.keys() for row in dataset]						)
								_lowercase :    Union[str, Any]							=							{col: [row.get(lowerCamelCase_						) for row in dataset] for col in keys}
							else:
								_lowercase :    List[str]							=							dataset
							_lowercase :    int							=							pa.Table.from_pydict(lowerCamelCase_						)
							yield file_idx, self._cast_table(lowerCamelCase_						)

						# If the file has one json object per line
						else:
							with open(lowerCamelCase_			,					'rb'						) as f:
								_lowercase :    Union[str, Any]							=							0
								# Use block_size equal to the chunk size divided by 32 to leverage multithreading
								# Set a default minimum value of 16kB if the chunk size is really small
								_lowercase :    Dict							=							max(self.config.chunksize // 3_2			,					1_6 << 1_0						)
								_lowercase :    List[str]							=							(
								    self.config.encoding_errors if self.config.encoding_errors is not None else 'strict'
								)
								while True:
									_lowercase :    Optional[Any]							=							f.read(self.config.chunksize						)
									if not batch:
										break
									# Finish current line
									try:
										batch += f.readline()
									except (AttributeError, io.UnsupportedOperation):
										batch += readline(lowerCamelCase_						)
									# PyArrow only accepts utf-8 encoded bytes
									if self.config.encoding != "utf-8":
										_lowercase :    int							=							batch.decode(self.config.encoding			,					errors=lowerCamelCase_						).encode('utf-8'						)
									try:
										while True:
											try:
												_lowercase :    Union[str, Any]							=							paj.read_json(
												    io.BytesIO(lowerCamelCase_						)			,					read_options=paj.ReadOptions(block_size=lowerCamelCase_						)						)
												break
											except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
												if (
												    isinstance(lowerCamelCase_			,					pa.ArrowInvalid						)
												    and "straddling" not in str(lowerCamelCase_						)
												    or block_size > len(lowerCamelCase_						)
												):
													raise
												else:
													# Increase the block size in case it was too small.
													# The block size will be reset for the next file.
													logger.debug(
													    F'''Batch of {len(lowerCamelCase_						)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''						)
													block_size *= 2
									except pa.ArrowInvalid as e:
										try:
											with open(
											    lowerCamelCase_			,					encoding=self.config.encoding			,					errors=self.config.encoding_errors						) as f:
												_lowercase :    Dict							=							json.load(lowerCamelCase_						)
										except json.JSONDecodeError:
											logger.error(F'''Failed to read file \'{file}\' with error {type(lowerCamelCase_						)}: {e}'''						)
											raise e
										# If possible, parse the file as a list of json objects and exit the loop
										if isinstance(lowerCamelCase_			,					lowerCamelCase_						):  # list is the only sequence type supported in JSON
											try:
												_lowercase :    Dict							=							set().union(*[row.keys() for row in dataset]						)
												_lowercase :    Any							=							{col: [row.get(lowerCamelCase_						) for row in dataset] for col in keys}
												_lowercase :    int							=							pa.Table.from_pydict(lowerCamelCase_						)
											except (pa.ArrowInvalid, AttributeError) as e:
												logger.error(F'''Failed to read file \'{file}\' with error {type(lowerCamelCase_						)}: {e}'''						)
												raise ValueError(F'''Not able to read records in the JSON file at {file}.'''						) from None
											yield file_idx, self._cast_table(lowerCamelCase_						)
											break
										else:
											logger.error(F'''Failed to read file \'{file}\' with error {type(lowerCamelCase_						)}: {e}'''						)
											raise ValueError(
											    F'''Not able to read records in the JSON file at {file}. '''
											    F'''You should probably indicate the field of the JSON file containing your records. '''
											    F'''This JSON file contain the following fields: {str(list(dataset.keys()						)						)}. '''
											    F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''						) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
									yield (file_idx, batch_idx), self._cast_table(lowerCamelCase_						)
									batch_idx += 1




"""simple docstring"""







import os
from typing import Optional

import fsspec
from fsspec.archive import AbstractArchiveFileSystem
from fsspec.utils import DEFAULT_BLOCK_SIZE



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  ""
				_snake_case     =  (
				    None  # protocol passed in prefix to the url. ex: "gzip", for gzip://file.txt::http://foo.bar/file.txt.gz
				)
				_snake_case     =  None  # compression type in fsspec. ex: "gzip"
				_snake_case     =  None  # extension of the filename to strip. ex: "".gz" to get file.txt from file.txt.gz







				def __init__( self    :       str			,					lowerCamelCase_    :       str = ""			,					lowerCamelCase_    :       Optional[str] = None			,					lowerCamelCase_    :       Optional[dict] = None			,					**lowerCamelCase_    :       List[str]						):

					"""simple docstring"""






					super().__init__(self			,					**lowerCamelCase_						)
					# always open as "rb" since fsspec can then use the TextIOWrapper to make it work for "r" mode
					_lowercase :    Union[str, Any]							=							fsspec.open(
					    lowerCamelCase_			,					mode='rb'			,					protocol=lowerCamelCase_			,					compression=self.compression			,					client_kwargs={
					        'requote_redirect_url': False,  # see https://github.com/huggingface/datasets/pull/5459
					        'trust_env': True,  # Enable reading proxy env variables.
					        **(target_options or {}).pop('client_kwargs'			,					{}						),  # To avoid issues if it was already passed.
					    }			,					**(target_options or {})			,					)
					_lowercase :    str							=							os.path.basename(self.file.path.split('::'						)[0]						)
					_lowercase :    Optional[int]							=							(
					    self.compressed_name[: self.compressed_name.rindex('.'						)]
					    if '.' in self.compressed_name
					    else self.compressed_name
					)
					_lowercase :    str							=							None







				@classmethod
				def 							__UpperCAmelCase					( cls    :       int			,					lowerCamelCase_    :       List[str]						):

					"""simple docstring"""






					return super()._strip_protocol(lowerCamelCase_						).lstrip('/'						)







				def 							__UpperCAmelCase					( self    :       Optional[Any]						):

					"""simple docstring"""






					if self.dir_cache is None:
						_lowercase :    Tuple							=							{**self.file.fs.info(self.file.path						), 'name': self.uncompressed_name}
						_lowercase :    int							=							{f['name']: f}







				def 							__UpperCAmelCase					( self    :       List[str]			,					lowerCamelCase_    :       str						):

					"""simple docstring"""






					return self.file.open().read()







				def 							__UpperCAmelCase					( self    :       Dict			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       str = "rb"			,					lowerCamelCase_    :       str=None			,					lowerCamelCase_    :       str=True			,					lowerCamelCase_    :       Union[str, Any]=None			,					**lowerCamelCase_    :       Optional[int]			,					):

					"""simple docstring"""






					_lowercase :    Union[str, Any]							=							self._strip_protocol(lowerCamelCase_						)
					if mode != "rb":
						raise ValueError(F'''Tried to read with mode {mode} on file {self.file.path} opened with mode \'rb\''''						)
					return self.file.open()



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "bz2"
				_snake_case     =  "bz2"
				_snake_case     =  ".bz2"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "gzip"
				_snake_case     =  "gzip"
				_snake_case     =  ".gz"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "lz4"
				_snake_case     =  "lz4"
				_snake_case     =  ".lz4"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "xz"
				_snake_case     =  "xz"
				_snake_case     =  ".xz"



class  _lowerCamelCase			(__lowerCamelCase     ):
				_snake_case     =  "zstd"
				_snake_case     =  "zstd"
				_snake_case     =  ".zst"







				def __init__( self    :       Union[str, Any]			,					lowerCamelCase_    :       str			,					lowerCamelCase_    :       str = "rb"			,					lowerCamelCase_    :       Optional[str] = None			,					lowerCamelCase_    :       Optional[dict] = None			,					lowerCamelCase_    :       int = DEFAULT_BLOCK_SIZE			,					**lowerCamelCase_    :       int			,					):

					"""simple docstring"""






					super().__init__(
					    fo=lowerCamelCase_			,					mode=lowerCamelCase_			,					target_protocol=lowerCamelCase_			,					target_options=lowerCamelCase_			,					block_size=lowerCamelCase_			,					**lowerCamelCase_			,					)
					# We need to wrap the zstd decompressor to avoid this error in fsspec==2021.7.0 and zstandard==0.15.2:
					#
					# File "/Users/user/.virtualenvs/hf-datasets/lib/python3.7/site-packages/fsspec/core.py", line 145, in open
					#     out.close = close
					# AttributeError: 'zstd.ZstdDecompressionReader' object attribute 'close' is read-only
					#
					# see https://github.com/intake/filesystem_spec/issues/725
					_lowercase :    Any							=							self.file.__enter__



					class  _lowerCamelCase			:







									def __init__( self    :       Any			,					lowerCamelCase_    :       List[Any]						):

										"""simple docstring"""






										_lowercase :    Tuple							=							file_







									def __enter__( self    :       str						):

										"""simple docstring"""






										self._file.__enter__()
										return self







									def __exit__( self    :       Union[str, Any]			,					*lowerCamelCase_    :       Optional[Any]			,					**lowerCamelCase_    :       Union[str, Any]						):

										"""simple docstring"""






										self._file.__exit__(*lowerCamelCase_			,					**lowerCamelCase_						)







									def __iter__( self    :       Optional[int]						):

										"""simple docstring"""






										return iter(self._file						)







									def 							__UpperCAmelCase					( self    :       int						):

										"""simple docstring"""






										return next(self._file						)







									def __getattr__( self    :       List[str]			,					lowerCamelCase_    :       str						):

										"""simple docstring"""






										return getattr(self._file			,					lowerCamelCase_						)

					def fixed_enter(*lowerCamelCase_    :       List[Any]			,					**lowerCamelCase_    :       int						):
						return WrappedFile(_enter(*lowerCamelCase_			,					**lowerCamelCase_						)						)

					_lowercase :    Optional[int]							=							fixed_enter



