
"""simple docstring"""
import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


lowercase_       :						List[str]     					=	datasets.utils.logging.get_logger(__name__)
@dataclass
class 			UpperCamelCase      (							datasets.BuilderConfig  ):
     A__      =			10000
     A__      =			None
     A__      =			None
class 			UpperCamelCase      (							datasets.ArrowBasedBuilder  ):
     A__      =			ParquetConfig
     def 				__SCREAMING_SNAKE_CASE							(     self				):




       """simple docstring"""


       return datasets.DatasetInfo(features=self.config.features				)
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__				):




       """simple docstring"""


       if not self.config.data_files:
         raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''				)
       _SCREAMING_SNAKE_CASE   :				List[Any]							    = dl_manager.download_and_extract(self.config.data_files				)
       if isinstance(snake_case__  ,     (str, list, tuple)				):
         _SCREAMING_SNAKE_CASE   :				int							    = data_files
         if isinstance(snake_case__  ,     snake_case__				):
           _SCREAMING_SNAKE_CASE   :				List[Any]							    = [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = [dl_manager.iter_files(snake_case__				) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,     gen_kwargs={"files": files}				)]
       _SCREAMING_SNAKE_CASE   :				Tuple							    = []
       for split_name, files in data_files.items():
         if isinstance(snake_case__  ,     snake_case__				):
           _SCREAMING_SNAKE_CASE   :				Optional[int]							    = [files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         _SCREAMING_SNAKE_CASE   :				Tuple							    = [dl_manager.iter_files(snake_case__				) for file in files]
         # Infer features is they are stoed in the arrow schema
         if self.info.features is None:
           for file in itertools.chain.from_iterable(snake_case__				):
             with open(snake_case__  ,     "rb"				) as f:
               _SCREAMING_SNAKE_CASE   :				Optional[int]							    = datasets.Features.from_arrow_schema(pq.read_schema(snake_case__				)				)
             break
         splits.append(datasets.SplitGenerator(name=snake_case__  ,     gen_kwargs={"files": files}				)				)
       return splits
     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__				):




       """simple docstring"""


       if self.info.features is not None:
         # more expensive cast to support nested features with keys in a different order
         # allows str <-> int/float or str to Audio for example
         _SCREAMING_SNAKE_CASE   :				Union[str, Any]							    = table_cast(snake_case__  ,     self.info.features.arrow_schema				)
       return pa_table




     def 				__SCREAMING_SNAKE_CASE							(     self  ,     snake_case__				):




       """simple docstring"""


       _SCREAMING_SNAKE_CASE   :				Any							    = self.info.features.arrow_schema if self.info.features is not None else None
       if self.info.features is not None and self.config.columns is not None:
         if sorted(field.name for field in schema				) != sorted(self.config.columns				):
           raise ValueError(
               F'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''				)
       for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case__				)				):
         with open(snake_case__  ,     "rb"				) as f:
           _SCREAMING_SNAKE_CASE   :				List[Any]							    = pq.ParquetFile(snake_case__				)
           try:
             for batch_idx, record_batch in enumerate(
                 parquet_file.iter_batches(batch_size=self.config.batch_size  ,     columns=self.config.columns				)				):
               _SCREAMING_SNAKE_CASE   :				int							    = pa.Table.from_batches([record_batch]				)
               # Uncomment for debugging (will print the Arrow table size and elements)
               # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
               # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
               yield F'''{file_idx}_{batch_idx}''', self._cast_table(snake_case__				)
           except ValueError as e:
             logger.error(F'''Failed to read file \'{file}\' with error {type(snake_case__				)}: {e}'''				)
             raise





"""simple docstring"""
import os

import pytest
import yaml

from datasets.features.features import Features, Value
from datasets.info import DatasetInfo, DatasetInfosDict





@pytest.mark.parametrize(
    "files",						[
        ["full:README.md", "dataset_infos.json"],
        ["empty:README.md", "dataset_infos.json"],
        ["dataset_infos.json"],
        ["full:README.md"],
    ],						)
def    _lowerCAmelCase    (     lowerCamelCase__							:							str,						lowerCamelCase__							:							str				)  ->    Optional[int]:
  _SCREAMING_SNAKE_CASE   :				Optional[int]							    = tmp_path_factory.mktemp("dset_infos_dir"				)
  if "full:README.md" in files:
    with open(dataset_infos_dir / "README.md",						"w"				) as f:
      f.write("---\ndataset_info:\n  dataset_size: 42\n---"				)
  if "empty:README.md" in files:
    with open(dataset_infos_dir / "README.md",						"w"				) as f:
      f.write(""				)
    # we want to support dataset_infos.json for backward compatibility
  if "dataset_infos.json" in files:
    with open(dataset_infos_dir / "dataset_infos.json",						"w"				) as f:
      f.write("{\"default\": {\"dataset_size\": 42}}"				)
  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = DatasetInfosDict.from_directory(lowerCamelCase__				)
  assert dataset_infos
  assert dataset_infos["default"].dataset_size == 4_2





@pytest.mark.parametrize(
    "dataset_info",						[
        DatasetInfo(),
        DatasetInfo(
            description="foo",						features=Features({"a": Value("int32"				)}				),						builder_name="builder",						config_name="config",						version="1.0.0",						splits=[{"name": "train"}],						download_size=4_2,						),
    ],						)
def    _lowerCAmelCase    (     lowerCamelCase__							:							Union[str, Any],						lowerCamelCase__							:							DatasetInfo				)  ->    List[Any]:
  _SCREAMING_SNAKE_CASE   :				Dict							    = str(lowerCamelCase__				)
  dataset_info.write_to_directory(lowerCamelCase__				)
  _SCREAMING_SNAKE_CASE   :				int							    = DatasetInfo.from_directory(lowerCamelCase__				)
  assert dataset_info == reloaded
  assert os.path.exists(os.path.join(lowerCamelCase__,						"dataset_info.json"				)				)





def    _lowerCAmelCase    (     )  ->    Union[str, Any]:
  _SCREAMING_SNAKE_CASE   :				str							    = DatasetInfo(
      description="foo",						citation="bar",						homepage="https://foo.bar",						license="CC0",						features=Features({"a": Value("int32"				)}				),						post_processed={},						supervised_keys=(),						task_templates=[],						builder_name="builder",						config_name="config",						version="1.0.0",						splits=[{"name": "train", "num_examples": 4_2}],						download_checksums={},						download_size=1_3_3_7,						post_processing_size=4_4_2,						dataset_size=1_2_3_4,						size_in_bytes=1_3_3_7 + 4_4_2 + 1_2_3_4,						)
  _SCREAMING_SNAKE_CASE   :				List[str]							    = dataset_info._to_yaml_dict()
  assert sorted(lowerCamelCase__				) == sorted(DatasetInfo._INCLUDED_INFO_IN_YAML				)
  for key in DatasetInfo._INCLUDED_INFO_IN_YAML:
    assert key in dataset_info_yaml_dict
    assert isinstance(dataset_info_yaml_dict[key],						(list, dict, int, str)				)
  _SCREAMING_SNAKE_CASE   :				Optional[Any]							    = yaml.safe_dump(lowerCamelCase__				)
  _SCREAMING_SNAKE_CASE   :				List[Any]							    = yaml.safe_load(lowerCamelCase__				)
  assert dataset_info_yaml_dict == reloaded





def    _lowerCAmelCase    (     )  ->    Tuple:
  _SCREAMING_SNAKE_CASE   :				Dict							    = DatasetInfo()
  _SCREAMING_SNAKE_CASE   :				List[Any]							    = dataset_info._to_yaml_dict()
  assert dataset_info_yaml_dict == {}





@pytest.mark.parametrize(
    "dataset_infos_dict",						[
        DatasetInfosDict(),
        DatasetInfosDict({"default": DatasetInfo()}				),
        DatasetInfosDict({"my_config_name": DatasetInfo()}				),
        DatasetInfosDict(
            {
                "default": DatasetInfo(
                    description="foo",						features=Features({"a": Value("int32"				)}				),						builder_name="builder",						config_name="config",						version="1.0.0",						splits=[{"name": "train"}],						download_size=4_2,						)
            }				),
        DatasetInfosDict(
            {
                "v1": DatasetInfo(dataset_size=4_2				),
                "v2": DatasetInfo(dataset_size=1_3_3_7				),
            }				),
    ],						)
def    _lowerCAmelCase    (     lowerCamelCase__							:							Any,						lowerCamelCase__							:							DatasetInfosDict				)  ->    Optional[int]:
  _SCREAMING_SNAKE_CASE   :				str							    = str(lowerCamelCase__				)
  dataset_infos_dict.write_to_directory(lowerCamelCase__				)
  _SCREAMING_SNAKE_CASE   :				str							    = DatasetInfosDict.from_directory(lowerCamelCase__				)

  # the config_name of the dataset_infos_dict take over the attribute
  for config_name, dataset_info in dataset_infos_dict.items():
    _SCREAMING_SNAKE_CASE   :				List[Any]							    = config_name
    # the yaml representation doesn't include fields like description or citation
    # so we just test that we can recover what we can from the yaml
    _SCREAMING_SNAKE_CASE   :				Dict							    = DatasetInfo._from_yaml_dict(dataset_info._to_yaml_dict()				)
  assert dataset_infos_dict == reloaded

  if dataset_infos_dict:
    assert os.path.exists(os.path.join(lowerCamelCase__,						"README.md"				)				)



