
"""simple docstring"""



import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


UpperCamelCase__          =	datasets.utils.logging.get_logger(__name__)





@dataclass
class 				a			(					datasets.BuilderConfig      ):
      UpperCamelCase							:   int           =     1_0_0_0_0
      UpperCamelCase							:   Optional[List[str]]           =     None
      UpperCamelCase							:   Optional[datasets.Features]           =     None





class 				a			(					datasets.ArrowBasedBuilder      ):
      UpperCamelCase							:   Tuple           =     ParquetConfig



      def  __snake_case							(   self      ):
          return datasets.DatasetInfo(features=self.config.features      )



      def  __snake_case							(   self     ,	UpperCamelCase_      ):
          if not self.config.data_files:
              raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''      )
          UpperCAmelCase__	:      Any     							=					dl_manager.download_and_extract(self.config.data_files      )
          if isinstance(UpperCamelCase_     ,	(str, list, tuple)      ):
              UpperCAmelCase__	:      Any     							=					data_files
              if isinstance(UpperCamelCase_     ,	UpperCamelCase_      ):
                  UpperCAmelCase__	:      List[str]     							=					[files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              UpperCAmelCase__	:      Union[str, Any]     							=					[dl_manager.iter_files(UpperCamelCase_      ) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,	gen_kwargs={'files': files}      )]
          UpperCAmelCase__	:      Tuple     							=					[]
          for split_name, files in data_files.items():
              if isinstance(UpperCamelCase_     ,	UpperCamelCase_      ):
                  UpperCAmelCase__	:      Tuple     							=					[files]
              # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
              UpperCAmelCase__	:      Optional[int]     							=					[dl_manager.iter_files(UpperCamelCase_      ) for file in files]
              # Infer features is they are stoed in the arrow schema
              if self.info.features is None:
                  for file in itertools.chain.from_iterable(UpperCamelCase_      ):
                      with open(UpperCamelCase_     ,	'rb'      ) as f:
                          UpperCAmelCase__	:      List[str]     							=					datasets.Features.from_arrow_schema(pq.read_schema(UpperCamelCase_      )      )
                      break
              splits.append(datasets.SplitGenerator(name=UpperCamelCase_     ,	gen_kwargs={'files': files}      )      )
          return splits



      def  __snake_case							(   self     ,	UpperCamelCase_      ):
          if self.info.features is not None:
              # more expensive cast to support nested features with keys in a different order
              # allows str <-> int/float or str to Audio for example
              UpperCAmelCase__	:      List[Any]     							=					table_cast(UpperCamelCase_     ,	self.info.features.arrow_schema      )
          return pa_table



      def  __snake_case							(   self     ,	UpperCamelCase_      ):
          UpperCAmelCase__	:      Optional[int]     							=					self.info.features.arrow_schema if self.info.features is not None else None
          if self.info.features is not None and self.config.columns is not None:
              if sorted(field.name for field in schema      ) != sorted(self.config.columns      ):
                  raise ValueError(
                      F'''Tried to load parquet data with columns \'{self.config.columns}\' with mismatching features \'{self.info.features}\''''      )
          for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCamelCase_      )      ):
              with open(UpperCamelCase_     ,	'rb'      ) as f:
                  UpperCAmelCase__	:      Optional[int]     							=					pq.ParquetFile(UpperCamelCase_      )
                  try:
                      for batch_idx, record_batch in enumerate(
                          parquet_file.iter_batches(batch_size=self.config.batch_size     ,	columns=self.config.columns      )      ):
                          UpperCAmelCase__	:      Any     							=					pa.Table.from_batches([record_batch]      )
                          # Uncomment for debugging (will print the Arrow table size and elements)
                          # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                          # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                          yield F'''{file_idx}_{batch_idx}''', self._cast_table(UpperCamelCase_      )
                  except ValueError as e:
                      logger.error(F'''Failed to read file \'{file}\' with error {type(UpperCamelCase_      )}: {e}'''      )
                      raise



"""simple docstring"""



from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_torch_available,
)


UpperCamelCase__          =	{
    'configuration_mega': ['MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegaConfig', 'MegaOnnxConfig'],
}

try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    UpperCamelCase__          =	[
        'MEGA_PRETRAINED_MODEL_ARCHIVE_LIST',
        'MegaForCausalLM',
        'MegaForMaskedLM',
        'MegaForMultipleChoice',
        'MegaForQuestionAnswering',
        'MegaForSequenceClassification',
        'MegaForTokenClassification',
        'MegaModel',
        'MegaPreTrainedModel',
    ]

if TYPE_CHECKING:
    from .configuration_mega import MEGA_PRETRAINED_CONFIG_ARCHIVE_MAP, MegaConfig, MegaOnnxConfig

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_mega import (
            MEGA_PRETRAINED_MODEL_ARCHIVE_LIST,
            MegaForCausalLM,
            MegaForMaskedLM,
            MegaForMultipleChoice,
            MegaForQuestionAnswering,
            MegaForSequenceClassification,
            MegaForTokenClassification,
            MegaModel,
            MegaPreTrainedModel,
        )

else:
    import sys

    UpperCamelCase__          =	_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)

