



import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


__A							       =		datasets.utils.logging.get_logger(__name__)






@dataclass
class 						lowercase		(					datasets.BuilderConfig):






	"""simple docstring"""


	a__ :   Any						  =				1_0000
	a__ :   List[str]						  =				None
	a__ :   int						  =				None






class 						lowercase		(					datasets.ArrowBasedBuilder):






	"""simple docstring"""


	a__ :   List[Any]						  =				ParquetConfig




	def     _SCREAMING_SNAKE_CASE						(      self      :		Optional[Any]	)     ->					Optional[int]:
							return datasets.DatasetInfo(features=self.config.features	)




	def     _SCREAMING_SNAKE_CASE						(      self      :		int		,    __UpperCAmelCase      :		Dict	)     ->					Optional[int]:
							if not self.config.data_files:
													raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""	)
							UpperCAmelCase_=  dl_manager.download_and_extract(self.config.data_files	)
							if isinstance(__UpperCAmelCase		,    (str, list, tuple)	):
													UpperCAmelCase_=  data_files
													if isinstance(__UpperCAmelCase		,    __UpperCAmelCase	):
																			UpperCAmelCase_=  [files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													UpperCAmelCase_=  [dl_manager.iter_files(__UpperCAmelCase	) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,    gen_kwargs={"""files""": files}	)]
							UpperCAmelCase_=  []
							for split_name, files in data_files.items():
													if isinstance(__UpperCAmelCase		,    __UpperCAmelCase	):
																			UpperCAmelCase_=  [files]
													# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
													UpperCAmelCase_=  [dl_manager.iter_files(__UpperCAmelCase	) for file in files]
													# Infer features is they are stoed in the arrow schema
													if self.info.features is None:
																			for file in itertools.chain.from_iterable(__UpperCAmelCase	):
																									with open(__UpperCAmelCase		,    """rb"""	) as f:
																															UpperCAmelCase_=  datasets.Features.from_arrow_schema(pq.read_schema(__UpperCAmelCase	)	)
																									break
													splits.append(datasets.SplitGenerator(name=__UpperCAmelCase		,    gen_kwargs={"""files""": files}	)	)
							return splits




	def     _SCREAMING_SNAKE_CASE						(      self      :		Any		,    __UpperCAmelCase      :		Dict	)     ->					pa.Table:
							if self.info.features is not None:
													# more expensive cast to support nested features with keys in a different order
													# allows str <-> int/float or str to Audio for example
													UpperCAmelCase_=  table_cast(__UpperCAmelCase		,    self.info.features.arrow_schema	)
							return pa_table




	def     _SCREAMING_SNAKE_CASE						(      self      :		Any		,    __UpperCAmelCase      :		int	)     ->					List[Any]:
							UpperCAmelCase_=  self.info.features.arrow_schema if self.info.features is not None else None
							if self.info.features is not None and self.config.columns is not None:
													if sorted(field.name for field in schema	) != sorted(self.config.columns	):
																			raise ValueError(
																			    F"""Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\'"""	)
							for file_idx, file in enumerate(itertools.chain.from_iterable(__UpperCAmelCase	)	):
													with open(__UpperCAmelCase		,    """rb"""	) as f:
																			UpperCAmelCase_=  pq.ParquetFile(__UpperCAmelCase	)
																			try:
																									for batch_idx, record_batch in enumerate(
																									    parquet_file.iter_batches(batch_size=self.config.batch_size		,    columns=self.config.columns	)	):
																															UpperCAmelCase_=  pa.Table.from_batches([record_batch]	)
																															# Uncomment for debugging (will print the Arrow table size and elements)
																															# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																															# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield F"""{file_idx}_{batch_idx}""", self._cast_table(__UpperCAmelCase	)
																			except ValueError as e:
																									logger.error(F"""Failed to read file \'{file}\' with error {type(__UpperCAmelCase	)}: {e}"""	)
																									raise












from dataclasses import dataclass, field
from typing import ClassVar, Dict

from ..features import Features, Value
from .base import TaskTemplate




@dataclass(frozen=snake_case_							)
class 					UpperCamelCase_ (     snake_case_							):



	'''simple docstring'''




	lowerCAmelCase						=					field(default='''language-modeling'''			,   metadata={'''include_in_asdict_even_if_is_default''': True}							)

	lowerCAmelCase						=					Features({'''text''': Value('''string'''							)}							)
	lowerCAmelCase						=					Features({}							)
	lowerCAmelCase						=					"text"




	@property
	def 				_UpperCamelCase     (			self   )			->       Dict[str, str]:
								return {self.text_column: "text"}





