




a_						     =    [
    "VerificationMode",
    "Version",
    "disable_progress_bar",
    "enable_progress_bar",
    "is_progress_bar_enabled",
    "experimental",
]

from .info_utils import VerificationMode
from .logging import disable_progress_bar, enable_progress_bar, is_progress_bar_enabled
from .version import Version
from .experimental import experimental


import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


a_						     =    datasets.utils.logging.get_logger(__name__)






@dataclass
class _UpperCamelCase							(							datasets.BuilderConfig   ):
			'''simple docstring'''







			lowerCamelCase__           =10000
			lowerCamelCase__           =None
			lowerCamelCase__           =None






class _UpperCamelCase							(							datasets.ArrowBasedBuilder   ):
			'''simple docstring'''







			lowerCamelCase__           =ParquetConfig
			def 							__UpperCamelCase (			self							: str	)							->       str:

				"""simple docstring"""

				return datasets.DatasetInfo(features=self.config.features	)
			def 							__UpperCamelCase (			self							: Dict      ,      a							: List[Any]	)							->       Tuple:

				"""simple docstring"""

				if not self.config.data_files:
					raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"	)
				SCREAMING_SNAKE_CASE   :      Tuple				     =       dl_manager.download_and_extract(self.config.data_files	)
				if isinstance(a      ,      (str, list, tuple)	):
					SCREAMING_SNAKE_CASE   :      Dict				     =       data_files
					if isinstance(a      ,      a	):
						SCREAMING_SNAKE_CASE   :      Union[str, Any]				     =       [files]
					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
					SCREAMING_SNAKE_CASE   :      Optional[Any]				     =       [dl_manager.iter_files(a	) for file in files]
					return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,      gen_kwargs={"files": files}	)]
				SCREAMING_SNAKE_CASE   :      str				     =       []
				for split_name, files in data_files.items():
					if isinstance(a      ,      a	):
						SCREAMING_SNAKE_CASE   :      Optional[int]				     =       [files]
					# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
					SCREAMING_SNAKE_CASE   :      Tuple				     =       [dl_manager.iter_files(a	) for file in files]
					# Infer features is they are stoed in the arrow schema
					if self.info.features is None:
						for file in itertools.chain.from_iterable(a	):
							with open(a      ,      "rb"	) as f:
								SCREAMING_SNAKE_CASE   :      Dict				     =       datasets.Features.from_arrow_schema(pq.read_schema(a	)	)
							break
					splits.append(datasets.SplitGenerator(name=a      ,      gen_kwargs={"files": files}	)	)
				return splits
			def 							__UpperCamelCase (			self							: Dict      ,      a							: pa.Table	)							->       pa.Table:

				"""simple docstring"""

				if self.info.features is not None:
					# more expensive cast to support nested features with keys in a different order
					# allows str <-> int/float or str to Audio for example
					SCREAMING_SNAKE_CASE   :      str				     =       table_cast(a      ,      self.info.features.arrow_schema	)
				return pa_table






			def 							__UpperCamelCase (			self							: List[str]      ,      a							: Optional[Any]	)							->       Optional[Any]:

				"""simple docstring"""

				SCREAMING_SNAKE_CASE   :      Any				     =       self.info.features.arrow_schema if self.info.features is not None else None
				if self.info.features is not None and self.config.columns is not None:
					if sorted(field.name for field in schema	) != sorted(self.config.columns	):
						raise ValueError(
						    F"Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"	)
				for file_idx, file in enumerate(itertools.chain.from_iterable(a	)	):
					with open(a      ,      "rb"	) as f:
						SCREAMING_SNAKE_CASE   :      Optional[int]				     =       pq.ParquetFile(a	)
						try:
							for batch_idx, record_batch in enumerate(
							    parquet_file.iter_batches(batch_size=self.config.batch_size      ,      columns=self.config.columns	)	):
								SCREAMING_SNAKE_CASE   :      int				     =       pa.Table.from_batches([record_batch]	)
								# Uncomment for debugging (will print the Arrow table size and elements)
								# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
								# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
								yield F"{file_idx}_{batch_idx}", self._cast_table(a	)
						except ValueError as e:
							logger.error(F"Failed to read file '{file}' with error {type(a	)}: {e}"	)
							raise