'''simple docstring'''





import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader




class  __magic_name__  (		__a    ):





   """simple docstring"""




   def __init__(	self					:  Optional[int]		,  _lowercase					:  Tuple		,  _lowercase					:  Tuple = None		,  _lowercase					:  Optional[int] = None		,  _lowercase					:  Union[str, Any] = None		,  _lowercase					:  int = False		,  _lowercase					:  int = False		,  _lowercase					:  Any = None		,  _lowercase					:  Any = None		,  **_lowercase					:  Tuple		,  ):



    """simple docstring"""



    super().__init__(
        __lowerCAmelCase		,  split=__lowerCAmelCase		,  features=__lowerCAmelCase		,  cache_dir=__lowerCAmelCase		,  keep_in_memory=__lowerCAmelCase		,  streaming=__lowerCAmelCase		,  num_proc=__lowerCAmelCase		,  **__lowerCAmelCase		,  )
    _UpperCamelCase:     Tuple							= field
    _UpperCamelCase:     Optional[int]							= path_or_paths if isinstance(__lowerCAmelCase		,  __lowerCAmelCase							) else {self.split: path_or_paths}
    _UpperCamelCase:     Dict							= Json(
        cache_dir=__lowerCAmelCase		,  data_files=__lowerCAmelCase		,  features=__lowerCAmelCase		,  field=__lowerCAmelCase		,  **__lowerCAmelCase		,  )



   def   lowerCAmelCase						(	self					:  Any							):



    """simple docstring"""



    if self.streaming:
     _UpperCamelCase:     Any							= self.builder.as_streaming_dataset(split=self.split							)
    # Build regular (map-style) dataset
    else:
     _UpperCamelCase:     List[str]							= None
     _UpperCamelCase:     Optional[int]							= None
     _UpperCamelCase:     int							= None
     _UpperCamelCase:     int							= None

     self.builder.download_and_prepare(
         download_config=__lowerCAmelCase		,  download_mode=__lowerCAmelCase		,  verification_mode=__lowerCAmelCase		,  base_path=__lowerCAmelCase		,  num_proc=self.num_proc		,  )
     _UpperCamelCase:     str							= self.builder.as_dataset(
         split=self.split		,  verification_mode=__lowerCAmelCase		,  in_memory=self.keep_in_memory							)
    return dataset




class  __magic_name__  :





   """simple docstring"""




   def __init__(	self					:  Optional[int]		,  _lowercase					:  str		,  _lowercase					:  List[str]		,  _lowercase					:  Union[str, Any] = None		,  _lowercase					:  Any = None		,  **_lowercase					:  Union[str, Any]		,  ):



    """simple docstring"""



    if num_proc is not None and num_proc <= 0:
     raise ValueError(f"""num_proc {num_proc} must be an integer > 0."""							)

    _UpperCamelCase:     List[str]							= dataset
    _UpperCamelCase:     Any							= path_or_buf
    _UpperCamelCase:     int							= batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
    _UpperCamelCase:     Dict							= num_proc
    _UpperCamelCase:     Union[str, Any]							= '''utf-8'''
    _UpperCamelCase:     Union[str, Any]							= to_json_kwargs



   def   lowerCAmelCase						(	self					:  List[str]							):



    """simple docstring"""



    _UpperCamelCase:     Union[str, Any]							= self.to_json_kwargs.pop('''path_or_buf'''		,  __lowerCAmelCase							)
    _UpperCamelCase:     Optional[int]							= self.to_json_kwargs.pop('''orient'''		,  '''records'''							)
    _UpperCamelCase:     Dict							= self.to_json_kwargs.pop('''lines'''		,  True if orient == '''records''' else False							)
    _UpperCamelCase:     str							= self.to_json_kwargs.pop('''index'''		,  False if orient in ['''split''', '''table'''] else True							)
    _UpperCamelCase:     int							= self.to_json_kwargs.pop('''compression'''		,  __lowerCAmelCase							)

    if compression not in [None, "infer", "gzip", "bz2", "xz"]:
     raise NotImplementedError(f"""`datasets` currently does not support {compression} compression"""							)

    if isinstance(self.path_or_buf		,  (str, bytes, os.PathLike)							):
     with fsspec.open(self.path_or_buf		,  '''wb'''		,  compression=__lowerCAmelCase							) as buffer:
      _UpperCamelCase:     Dict							= self._write(file_obj=__lowerCAmelCase		,  orient=__lowerCAmelCase		,  lines=__lowerCAmelCase		,  index=__lowerCAmelCase		,  **self.to_json_kwargs							)
    else:
     if compression:
      raise NotImplementedError(
          f"""The compression parameter is not supported when writing to a buffer, but compression={compression}"""
          ''' was passed. Please provide a local path instead.'''							)
     _UpperCamelCase:     Union[str, Any]							= self._write(
         file_obj=self.path_or_buf		,  orient=__lowerCAmelCase		,  lines=__lowerCAmelCase		,  index=__lowerCAmelCase		,  **self.to_json_kwargs							)
    return written



   def   lowerCAmelCase						(	self					:  Dict		,  _lowercase					:  List[str]							):



    """simple docstring"""



    _UpperCamelCase	,							_UpperCamelCase	,							_UpperCamelCase	,							_UpperCamelCase	,							_UpperCamelCase:     str							= args

    _UpperCamelCase:     Dict							= query_table(
        table=self.dataset.data		,  key=slice(__lowerCAmelCase		,  offset + self.batch_size							)		,  indices=self.dataset._indices		,  )
    _UpperCamelCase:     str							= batch.to_pandas().to_json(
        path_or_buf=__lowerCAmelCase		,  orient=__lowerCAmelCase		,  lines=__lowerCAmelCase		,  index=__lowerCAmelCase		,  **__lowerCAmelCase							)
    if not json_str.endswith('''\n'''							):
     json_str += "\n"
    return json_str.encode(self.encoding							)



   def   lowerCAmelCase						(	self					:  Union[str, Any]		,  _lowercase					:  int		,  _lowercase					:  Tuple		,  _lowercase					:  int		,  _lowercase					:  List[str]		,  **_lowercase					:  Union[str, Any]		,  ):



    """simple docstring"""



    _UpperCamelCase:     Union[str, Any]							= 0

    if self.num_proc is None or self.num_proc == 1:
     for offset in logging.tqdm(
         range(0		,  len(self.dataset							)		,  self.batch_size							)		,  unit='''ba'''		,  disable=not logging.is_progress_bar_enabled()		,  desc='''Creating json from Arrow format'''		,  ):
      _UpperCamelCase:     Dict							= self._batch_json((offset, orient, lines, index, to_json_kwargs)							)
      written += file_obj.write(__lowerCAmelCase							)
    else:
     _UpperCamelCase	,							_UpperCamelCase:     Dict							= len(self.dataset							), self.batch_size
     with multiprocessing.Pool(self.num_proc							) as pool:
      for json_str in logging.tqdm(
          pool.imap(
              self._batch_json		,  [(offset, orient, lines, index, to_json_kwargs) for offset in range(0		,  __lowerCAmelCase		,  __lowerCAmelCase							)]		,  )		,  total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size		,  unit='''ba'''		,  disable=not logging.is_progress_bar_enabled()		,  desc='''Creating json from Arrow format'''		,  ):
       written += file_obj.write(__lowerCAmelCase							)

    return written
# Lint as: python3
# pylint: enable=line-too-long
# pylint: disable=g-import-not-at-top,g-bad-import-order,wrong-import-position

UpperCAmelCase_			    =							'''2.13.1'''

import platform

import pyarrow
from packaging import version


if version.parse(platform.python_version()) < version.parse('''3.7'''):
  raise ImportWarning(
      '''To use `datasets`, Python>=3.7 is required, and the current version of Python doesn\'t match this condition.'''
  )

if version.parse(pyarrow.__version__).major < 8:
  raise ImportWarning(
      '''To use `datasets`, the module `pyarrow>=8.0.0` is required, and the current version of `pyarrow` doesn\'t match this condition.\n'''
      '''If you are running this in a Google Colab, you should probably just restart the runtime to use the right version of `pyarrow`.'''
  )

del platform
del pyarrow
del version

from .arrow_dataset import Dataset
from .arrow_reader import ReadInstruction
from .builder import ArrowBasedBuilder, BeamBasedBuilder, BuilderConfig, DatasetBuilder, GeneratorBasedBuilder
from .combine import concatenate_datasets, interleave_datasets
from .dataset_dict import DatasetDict, IterableDatasetDict
from .download import *
from .features import *
from .fingerprint import disable_caching, enable_caching, is_caching_enabled, set_caching_enabled
from .info import DatasetInfo, MetricInfo
from .inspect import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
    list_datasets,
    list_metrics,
)
from .iterable_dataset import IterableDataset
from .load import load_dataset, load_dataset_builder, load_from_disk, load_metric
from .metric import Metric
from .splits import (
    NamedSplit,
    NamedSplitAll,
    Split,
    SplitBase,
    SplitDict,
    SplitGenerator,
    SplitInfo,
    SubSplitInfo,
    percent,
)
from .tasks import *
from .utils import *
from .utils import logging


# deprecated modules
from datasets import arrow_dataset as _arrow_dataset  # isort:skip
from datasets import utils as _utils  # isort:skip
from datasets.utils import download_manager as _deprecated_download_manager  # isort:skip

UpperCAmelCase_			    =							concatenate_datasets
UpperCAmelCase_			    =							DownloadConfig
UpperCAmelCase_			    =							DownloadManager
UpperCAmelCase_			    =							DownloadMode
UpperCAmelCase_			    =							DownloadConfig
UpperCAmelCase_			    =							DownloadMode
UpperCAmelCase_			    =							DownloadManager

del _arrow_dataset, _utils, _deprecated_download_manager