






import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


lowerCAmelCase_           = datasets.utils.logging.get_logger(__name__)
@dataclass
class      UpperCamelCase (				datasets.BuilderConfig ):







		"""simple docstring"""



		snake_case    =   1_0_0_0_0
		snake_case    =   None
		snake_case    =   None


class      UpperCamelCase (				datasets.ArrowBasedBuilder ):







		"""simple docstring"""



		snake_case    =   ParquetConfig
		def       A(  self		:       Dict		)    ->   Tuple:


						'''simple docstring'''
						return datasets.DatasetInfo(features=self.config.features		)
		def       A(  self		:       List[str]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]		)    ->   List[str]:


						'''simple docstring'''
						if not self.config.data_files:
										raise ValueError(f'At least one data file must be specified, but got data_files={self.config.data_files}'		)
						A	 =      dl_manager.download_and_extract(self.config.data_files		)
						if isinstance(_SCREAMING_SNAKE_CASE						,(str, list, tuple)		):
										A	 =      data_files
										if isinstance(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		):
														A	 =      [files]
										# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
										A	 =      [dl_manager.iter_files(_SCREAMING_SNAKE_CASE		) for file in files]
										return [datasets.SplitGenerator(name=datasets.Split.TRAIN						,gen_kwargs={'files': files}		)]
						A	 =      []
						for split_name, files in data_files.items():
										if isinstance(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		):
														A	 =      [files]
										# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
										A	 =      [dl_manager.iter_files(_SCREAMING_SNAKE_CASE		) for file in files]
										# Infer features is they are stoed in the arrow schema
										if self.info.features is None:
														for file in itertools.chain.from_iterable(_SCREAMING_SNAKE_CASE		):
																		with open(_SCREAMING_SNAKE_CASE						,'rb'		) as f:
																						A	 =      datasets.Features.from_arrow_schema(pq.read_schema(_SCREAMING_SNAKE_CASE		)		)
																		break
										splits.append(datasets.SplitGenerator(name=_SCREAMING_SNAKE_CASE						,gen_kwargs={'files': files}		)		)
						return splits
		def       A(  self		:       Union[str, Any]						,_SCREAMING_SNAKE_CASE		:       pa.Table		)    ->   pa.Table:


						'''simple docstring'''
						if self.info.features is not None:
										# more expensive cast to support nested features with keys in a different order
										# allows str <-> int/float or str to Audio for example
										A	 =      table_cast(_SCREAMING_SNAKE_CASE						,self.info.features.arrow_schema		)
						return pa_table







		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Tuple		)    ->   int:


						'''simple docstring'''
						A	 =      self.info.features.arrow_schema if self.info.features is not None else None
						if self.info.features is not None and self.config.columns is not None:
										if sorted(field.name for field in schema		) != sorted(self.config.columns		):
														raise ValueError(
														    f'Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''		)
						for file_idx, file in enumerate(itertools.chain.from_iterable(_SCREAMING_SNAKE_CASE		)		):
										with open(_SCREAMING_SNAKE_CASE						,'rb'		) as f:
														A	 =      pq.ParquetFile(_SCREAMING_SNAKE_CASE		)
														try:
																		for batch_idx, record_batch in enumerate(
																		    parquet_file.iter_batches(batch_size=self.config.batch_size						,columns=self.config.columns		)		):
																						A	 =      pa.Table.from_batches([record_batch]		)
																						# Uncomment for debugging (will print the Arrow table size and elements)
																						# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																						# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																						yield f'{file_idx}_{batch_idx}', self._cast_table(_SCREAMING_SNAKE_CASE		)
														except ValueError as e:
																		logger.error(f'Failed to read file \'{file}\' with error {type(_SCREAMING_SNAKE_CASE		)}: {e}'		)
																		raise













from collections import deque

from .hash_table import HashTable
class      UpperCamelCase (				snake_case__ ):







		"""simple docstring"""



		def __init__(  self		:       Any						,*_SCREAMING_SNAKE_CASE		:       Optional[int]						,**_SCREAMING_SNAKE_CASE		:       Optional[Any]		)    ->   int:


						'''simple docstring'''
						super().__init__(*_SCREAMING_SNAKE_CASE						,**_SCREAMING_SNAKE_CASE		)
		def       A(  self		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       Optional[Any]						,_SCREAMING_SNAKE_CASE		:       Tuple		)    ->   List[str]:


						'''simple docstring'''
						A	 =      deque([]		) if self.values[key] is None else self.values[key]
						self.values[key].appendleft(_SCREAMING_SNAKE_CASE		)
						A	 =      self.values[key]
		def       A(  self		:       Optional[Any]		)    ->   Union[str, Any]:


						'''simple docstring'''
						return (
						    sum(self.charge_factor - len(_SCREAMING_SNAKE_CASE		) for slot in self.values		)
						    / self.size_table
						    * self.charge_factor
						)







		def       A(  self		:       Dict						,_SCREAMING_SNAKE_CASE		:       Optional[int]						,_SCREAMING_SNAKE_CASE		:       str=None		)    ->   Union[str, Any]:


						'''simple docstring'''
						if not (
						    len(self.values[key]		) == self.charge_factor and self.values.count(_SCREAMING_SNAKE_CASE		) == 0
						):
										return key
						return super()._collision_resolution(_SCREAMING_SNAKE_CASE						,_SCREAMING_SNAKE_CASE		)





