
import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


lowerCamelCase  =datasets.utils.logging.get_logger(__name__)




@dataclass
class 					_lowerCamelCase			(   datasets.BuilderConfig			):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		 =	1_0000
     SCREAMING_SNAKE_CASE_		 =	None
     SCREAMING_SNAKE_CASE_		 =	None




class 					_lowerCamelCase			(   datasets.ArrowBasedBuilder			):





     """simple docstring"""







     SCREAMING_SNAKE_CASE_		 =	ParquetConfig






     def  __SCREAMING_SNAKE_CASE      (				self	)	-> Optional[int]:




           """simple docstring"""







           return datasets.DatasetInfo(features=self.config.features	)






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> List[str]:




           """simple docstring"""







           if not self.config.data_files:
                 raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''	)
           UpperCamelCase__     : Any											=					dl_manager.download_and_extract(self.config.data_files	)
           if isinstance(__SCREAMING_SNAKE_CASE    ,    (str, list, tuple)	):
                 UpperCamelCase__     : Optional[Any]											=					data_files
                 if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE	):
                       UpperCamelCase__     : Dict											=					[files]
                 # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                 UpperCamelCase__     : Optional[Any]											=					[dl_manager.iter_files(__SCREAMING_SNAKE_CASE	) for file in files]
                 return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,    gen_kwargs={'''files''': files}	)]
           UpperCamelCase__     : Optional[Any]											=					[]
           for split_name, files in data_files.items():
                 if isinstance(__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE	):
                       UpperCamelCase__     : Union[str, Any]											=					[files]
                 # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                 UpperCamelCase__     : Optional[Any]											=					[dl_manager.iter_files(__SCREAMING_SNAKE_CASE	) for file in files]
                 # Infer features is they are stoed in the arrow schema
                 if self.info.features is None:
                       for file in itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE	):
                             with open(__SCREAMING_SNAKE_CASE    ,    '''rb'''	) as f:
                                   UpperCamelCase__     : Union[str, Any]											=					datasets.Features.from_arrow_schema(pq.read_schema(__SCREAMING_SNAKE_CASE	)	)
                             break
                 splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE    ,    gen_kwargs={'''files''': files}	)	)
           return splits






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> pa.Table:




           """simple docstring"""







           if self.info.features is not None:
                 # more expensive cast to support nested features with keys in a different order
                 # allows str <-> int/float or str to Audio for example
                 UpperCamelCase__     : str											=					table_cast(__SCREAMING_SNAKE_CASE    ,    self.info.features.arrow_schema	)
           return pa_table






     def  __SCREAMING_SNAKE_CASE      (				self    ,    __SCREAMING_SNAKE_CASE	)	-> Optional[Any]:




           """simple docstring"""







           UpperCamelCase__     : List[str]											=					self.info.features.arrow_schema if self.info.features is not None else None
           if self.info.features is not None and self.config.columns is not None:
                 if sorted(field.name for field in schema	) != sorted(self.config.columns	):
                       raise ValueError(
                           F'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''	)
           for file_idx, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE	)	):
                 with open(__SCREAMING_SNAKE_CASE    ,    '''rb'''	) as f:
                       UpperCamelCase__     : Union[str, Any]											=					pq.ParquetFile(__SCREAMING_SNAKE_CASE	)
                       try:
                             for batch_idx, record_batch in enumerate(
                                 parquet_file.iter_batches(batch_size=self.config.batch_size    ,    columns=self.config.columns	)	):
                                   UpperCamelCase__     : Tuple											=					pa.Table.from_batches([record_batch]	)
                                   # Uncomment for debugging (will print the Arrow table size and elements)
                                   # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                   # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                   yield F'''{file_idx}_{batch_idx}''', self._cast_table(__SCREAMING_SNAKE_CASE	)
                       except ValueError as e:
                             logger.error(F'''Failed to read file \'{file}\' with error {type(__SCREAMING_SNAKE_CASE	)}: {e}'''	)
                             raise




import logging
import os
from logging import (
    CRITICAL,  # NOQA
    DEBUG,  # NOQA
    ERROR,  # NOQA
    FATAL,  # NOQA
    INFO,  # NOQA
    NOTSET,  # NOQA
    WARN,  # NOQA
    WARNING,  # NOQA
)
from typing import Optional

from tqdm import auto as tqdm_lib


lowerCamelCase  ={
    "debug": logging.DEBUG,
    "info": logging.INFO,
    "warning": logging.WARNING,
    "error": logging.ERROR,
    "critical": logging.CRITICAL,
}

lowerCamelCase  =logging.WARNING


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : int											=					os.getenv('''DATASETS_VERBOSITY'''			,		UpperCamelCase__		)
      if env_level_str:
            if env_level_str in log_levels:
                  return log_levels[env_level_str]
            else:
                  logging.getLogger().warning(
                      f'''Unknown option DATASETS_VERBOSITY={env_level_str}, '''
                      f'''has to be one of: { ", ".join(log_levels.keys()		) }'''		)
      return _default_log_level


def 				SCREAMING_SNAKE_CASE_   (      ):
      return __name__.split('''.'''		)[0]


def 				SCREAMING_SNAKE_CASE_   (      ):
      return logging.getLogger(_get_library_name()		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      # Apply our default configuration to the library root logger.
      UpperCamelCase__     : List[Any]											=					_get_library_root_logger()
      library_root_logger.setLevel(_get_default_logging_level()		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : List[Any]											=					_get_library_root_logger()
      library_root_logger.setLevel(logging.NOTSET		)


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__ = None		):
      if name is None:
            UpperCamelCase__     : Union[str, Any]											=					_get_library_name()
      return logging.getLogger(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return _get_library_root_logger().getEffectiveLevel()


def 				SCREAMING_SNAKE_CASE_   (      UpperCamelCase__		):
      _get_library_root_logger().setLevel(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      return set_verbosity(UpperCamelCase__		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : str											=					False


def 				SCREAMING_SNAKE_CASE_   (      ):
      UpperCamelCase__     : List[Any]											=					True


# Configure the library root logger at the module level (singleton-like)
_configure_library_root_logger()




class 					_lowerCamelCase			:





     """simple docstring"""







     def __init__(				self    ,    *__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)	-> int:  # pylint: disable=unused-argument




           """simple docstring"""







           UpperCamelCase__     : Dict											=					args[0] if args else None






     def __iter__(				self	)	-> Dict:




           """simple docstring"""







           return iter(self._iterator	)






     def __getattr__(				self    ,    __SCREAMING_SNAKE_CASE	)	-> Tuple:




           """simple docstring"""







           def empty_fn(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	):  # pylint: disable=unused-argument
                 return

           return empty_fn






     def __enter__(				self	)	-> Optional[int]:




           """simple docstring"""







           return self






     def __exit__(				self    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE	)	-> Dict:




           """simple docstring"""







           return
lowerCamelCase  =True




class 					_lowerCamelCase			:





     """simple docstring"""







     def __call__(				self    ,    *__SCREAMING_SNAKE_CASE    ,    __SCREAMING_SNAKE_CASE=False    ,    **__SCREAMING_SNAKE_CASE	)	-> int:




           """simple docstring"""







           if _tqdm_active and not disable:
                 return tqdm_lib.tqdm(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)
           else:
                 return EmptyTqdm(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)






     def  __SCREAMING_SNAKE_CASE      (				self    ,    *__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)	-> Tuple:




           """simple docstring"""







           UpperCamelCase__     : Any											=					None
           if _tqdm_active:
                 return tqdm_lib.tqdm.set_lock(*__SCREAMING_SNAKE_CASE    ,    **__SCREAMING_SNAKE_CASE	)






     def  __SCREAMING_SNAKE_CASE      (				self	)	-> Tuple:




           """simple docstring"""







           if _tqdm_active:
                 return tqdm_lib.tqdm.get_lock()
lowerCamelCase  =_tqdm_cls()


def 				SCREAMING_SNAKE_CASE_   (      ):
      global _tqdm_active
      return bool(_tqdm_active		)


def 				SCREAMING_SNAKE_CASE_   (      ):
      global _tqdm_active
      UpperCamelCase__     : Dict											=					True


def 				SCREAMING_SNAKE_CASE_   (      ):
      global _tqdm_active
      UpperCamelCase__     : Union[str, Any]											=					False


