import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowercase_  : Any								=					datasets.utils.logging.get_logger(__name__)






@dataclass
class 		_lowerCamelCase		( datasets.BuilderConfig     ):
    __a						=   None
    __a						=   "utf-8"
    __a						=   None
    __a						=   None
    __a						=   True  # deprecated
    __a						=   None  # deprecated
    __a						=   10 << 20  # 10MB
    __a						=   None






class 		_lowerCamelCase		( datasets.ArrowBasedBuilder     ):
    __a						=   JsonConfig
    def     UpperCamelCase_					(	self )							->       Union[str, Any]:
     if self.config.block_size is not None:
      logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead''' )
      SCREAMING_SNAKE_CASE__: Dict=   self.config.block_size
     if self.config.use_threads is not True:
      logger.warning(
          '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.''' )
     if self.config.newlines_in_values is not None:
      raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported''' )
     return datasets.DatasetInfo(features=self.config.features )
    def     UpperCamelCase_					(	self			,			lowerCAmelCase )							->       Union[str, Any]:
     if not self.config.data_files:
      raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}' )
     SCREAMING_SNAKE_CASE__: List[Any]=   dl_manager.download_and_extract(self.config.data_files )
     if isinstance(lowerCAmelCase			,			(str, list, tuple) ):
      SCREAMING_SNAKE_CASE__: List[Any]=   data_files
      if isinstance(lowerCAmelCase			,			lowerCAmelCase ):
       SCREAMING_SNAKE_CASE__: Optional[Any]=   [files]
      SCREAMING_SNAKE_CASE__: Optional[int]=   [dl_manager.iter_files(lowerCAmelCase ) for file in files]
      return [datasets.SplitGenerator(name=datasets.Split.TRAIN			,			gen_kwargs={'''files''': files} )]
     SCREAMING_SNAKE_CASE__: List[str]=   []
     for split_name, files in data_files.items():
      if isinstance(lowerCAmelCase			,			lowerCAmelCase ):
       SCREAMING_SNAKE_CASE__: Optional[int]=   [files]
      SCREAMING_SNAKE_CASE__: int=   [dl_manager.iter_files(lowerCAmelCase ) for file in files]
      splits.append(datasets.SplitGenerator(name=lowerCAmelCase			,			gen_kwargs={'''files''': files} ) )
     return splits
    def     UpperCamelCase_					(	self			,			lowerCAmelCase )							->       pa.Table:
     if self.config.features is not None:
      # adding missing columns
      for column_name in set(self.config.features ) - set(pa_table.column_names ):
       SCREAMING_SNAKE_CASE__: Any=   self.config.features.arrow_schema.field(lowerCAmelCase ).type
       SCREAMING_SNAKE_CASE__: str=   pa_table.append_column(lowerCAmelCase			,			pa.array([None] * len(lowerCAmelCase )			,			type=lowerCAmelCase ) )
      # more expensive cast to support nested structures with keys in a different order
      # allows str <-> int/float or str to Audio for example
      SCREAMING_SNAKE_CASE__: int=   table_cast(lowerCAmelCase			,			self.config.features.arrow_schema )
     return pa_table



    def     UpperCamelCase_					(	self			,			lowerCAmelCase )							->       Optional[int]:
     for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCAmelCase ) ):
      # If the file is one json object and if we need to look at the list of items in one specific field
      if self.config.field is not None:
       with open(lowerCAmelCase			,			encoding=self.config.encoding			,			errors=self.config.encoding_errors ) as f:
        SCREAMING_SNAKE_CASE__: str=   json.load(lowerCAmelCase )

       # We keep only the field we are interested in
       SCREAMING_SNAKE_CASE__: Any=   dataset[self.config.field]

       # We accept two format: a list of dicts or a dict of lists
       if isinstance(lowerCAmelCase			,			(list, tuple) ):
        SCREAMING_SNAKE_CASE__: int=   set().union(*[row.keys() for row in dataset] )
        SCREAMING_SNAKE_CASE__: Optional[int]=   {col: [row.get(lowerCAmelCase ) for row in dataset] for col in keys}
       else:
        SCREAMING_SNAKE_CASE__: List[str]=   dataset
       SCREAMING_SNAKE_CASE__: Optional[Any]=   pa.Table.from_pydict(lowerCAmelCase )
       yield file_idx, self._cast_table(lowerCAmelCase )

      # If the file has one json object per line
      else:
       with open(lowerCAmelCase			,			'''rb''' ) as f:
        SCREAMING_SNAKE_CASE__: Optional[Any]=   0
        # Use block_size equal to the chunk size divided by 32 to leverage multithreading
        # Set a default minimum value of 16kB if the chunk size is really small
        SCREAMING_SNAKE_CASE__: Tuple=   max(self.config.chunksize // 32			,			16 << 10 )
        SCREAMING_SNAKE_CASE__: List[Any]=   (
            self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
        )
        while True:
         SCREAMING_SNAKE_CASE__: Optional[int]=   f.read(self.config.chunksize )
         if not batch:
          break
         # Finish current line
         try:
          batch += f.readline()
         except (AttributeError, io.UnsupportedOperation):
          batch += readline(lowerCAmelCase )
         # PyArrow only accepts utf-8 encoded bytes
         if self.config.encoding != "utf-8":
          SCREAMING_SNAKE_CASE__: Dict=   batch.decode(self.config.encoding			,			errors=lowerCAmelCase ).encode('''utf-8''' )
         try:
          while True:
           try:
            SCREAMING_SNAKE_CASE__: List[str]=   paj.read_json(
                io.BytesIO(lowerCAmelCase )			,			read_options=paj.ReadOptions(block_size=lowerCAmelCase ) )
            break
           except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
            if (
                isinstance(lowerCAmelCase			,			pa.ArrowInvalid )
                and "straddling" not in str(lowerCAmelCase )
                or block_size > len(lowerCAmelCase )
            ):
             raise
            else:
             # Increase the block size in case it was too small.
             # The block size will be reset for the next file.
             logger.debug(
                 f'Batch of {len(lowerCAmelCase )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.' )
             block_size *= 2
         except pa.ArrowInvalid as e:
          try:
           with open(
               lowerCAmelCase			,			encoding=self.config.encoding			,			errors=self.config.encoding_errors ) as f:
            SCREAMING_SNAKE_CASE__: Tuple=   json.load(lowerCAmelCase )
          except json.JSONDecodeError:
           logger.error(f'Failed to read file \'{file}\' with error {type(lowerCAmelCase )}: {e}' )
           raise e
          # If possible, parse the file as a list of json objects and exit the loop
          if isinstance(lowerCAmelCase			,			lowerCAmelCase ):  # list is the only sequence type supported in JSON
           try:
            SCREAMING_SNAKE_CASE__: str=   set().union(*[row.keys() for row in dataset] )
            SCREAMING_SNAKE_CASE__: List[str]=   {col: [row.get(lowerCAmelCase ) for row in dataset] for col in keys}
            SCREAMING_SNAKE_CASE__: Dict=   pa.Table.from_pydict(lowerCAmelCase )
           except (pa.ArrowInvalid, AttributeError) as e:
            logger.error(f'Failed to read file \'{file}\' with error {type(lowerCAmelCase )}: {e}' )
            raise ValueError(f'Not able to read records in the JSON file at {file}.' ) from None
           yield file_idx, self._cast_table(lowerCAmelCase )
           break
          else:
           logger.error(f'Failed to read file \'{file}\' with error {type(lowerCAmelCase )}: {e}' )
           raise ValueError(
               f'Not able to read records in the JSON file at {file}. '
               f'You should probably indicate the field of the JSON file containing your records. '
               f'This JSON file contain the following fields: {str(list(dataset.keys() ) )}. '
               f'Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. ' ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
         yield (file_idx, batch_idx), self._cast_table(lowerCAmelCase )
         batch_idx += 1





'''simple docstring'''
from .testing import (
    are_the_same_tensors,
    execute_subprocess_async,
    require_bnb,
    require_cpu,
    require_cuda,
    require_huggingface_suite,
    require_mps,
    require_multi_gpu,
    require_multi_xpu,
    require_safetensors,
    require_single_gpu,
    require_single_xpu,
    require_torch_min_version,
    require_tpu,
    require_xpu,
    skip,
    slow,
)
from .training import RegressionDataset, RegressionModel, RegressionModelaXPU


from .scripts import test_script, test_sync, test_ops  # isort: skip





