

"""simple docstring"""


import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
	import sqlitea

	import sqlalchemy



class 			lowerCamelCase__						(   __a			):







						def __init__(							self   :    int			,			_lowercase   :    Optional[int]			,			_lowercase   :    int			,			_lowercase   :    str = None			,			_lowercase   :    Any = None			,			_lowercase   :    Optional[int] = False			,			**_lowercase   :    Dict			,			):
								super().__init__(features=lowerCAmelCase_			,			cache_dir=lowerCAmelCase_			,			keep_in_memory=lowerCAmelCase_			,			**lowerCAmelCase_		)
								A					       = Sql(
								    cache_dir=lowerCAmelCase_			,			features=lowerCAmelCase_			,			sql=lowerCAmelCase_			,			con=lowerCAmelCase_			,			**lowerCAmelCase_			,			)







						def __a						(							self   :    List[Any]		):
								A					       = None
								A					       = None
								A					       = None
								A					       = None

								self.builder.download_and_prepare(
								    download_config=lowerCAmelCase_			,			download_mode=lowerCAmelCase_			,			verification_mode=lowerCAmelCase_			,			base_path=lowerCAmelCase_			,			)

								# Build dataset for splits
								A					       = self.builder.as_dataset(
								    split='train'			,			verification_mode=lowerCAmelCase_			,			in_memory=self.keep_in_memory		)
								return dataset



class 			lowerCamelCase__						:







						def __init__(							self   :    Optional[Any]			,			_lowercase   :    List[str]			,			_lowercase   :    Any			,			_lowercase   :    Optional[Any]			,			_lowercase   :    Tuple = None			,			_lowercase   :    Optional[int] = None			,			**_lowercase   :    List[Any]			,			):







								if num_proc is not None and num_proc <= 0:
										raise ValueError(f'num_proc {num_proc} must be an integer > 0.'		)

								A					       = dataset
								A					       = name
								A					       = con
								A					       = batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
								A					       = num_proc
								A					       = to_sql_kwargs







						def __a						(							self   :    Optional[Any]		):
								A					       = self.to_sql_kwargs.pop('sql'			,			lowerCAmelCase_		)
								A					       = self.to_sql_kwargs.pop('con'			,			lowerCAmelCase_		)
								A					       = self.to_sql_kwargs.pop('index'			,			lowerCAmelCase_		)

								A					       = self._write(index=lowerCAmelCase_			,			**self.to_sql_kwargs		)
								return written







						def __a						(							self   :    Dict			,			_lowercase   :    Any		):
								A       , A       , A					       = args
								A					       = {**to_sql_kwargs, 'if_exists': 'append'} if offset > 0 else to_sql_kwargs
								A					       = query_table(
								    table=self.dataset.data			,			key=slice(lowerCAmelCase_			,			offset + self.batch_size		)			,			indices=self.dataset._indices			,			)
								A					       = batch.to_pandas()
								A					       = df.to_sql(self.name			,			self.con			,			index=lowerCAmelCase_			,			**lowerCAmelCase_		)
								return num_rows or len(lowerCAmelCase_		)







						def __a						(							self   :    Dict			,			_lowercase   :    List[Any]			,			**_lowercase   :    Dict		):
								A					       = 0

								if self.num_proc is None or self.num_proc == 1:
										for offset in logging.tqdm(
										    range(0			,			len(self.dataset		)			,			self.batch_size		)			,			unit='ba'			,			disable=not logging.is_progress_bar_enabled()			,			desc='Creating SQL from Arrow format'			,			):
												written += self._batch_sql((offset, index, to_sql_kwargs)		)
								else:
										A       , A					       = len(self.dataset		), self.batch_size
										with multiprocessing.Pool(self.num_proc		) as pool:
												for num_rows in logging.tqdm(
												    pool.imap(
												        self._batch_sql			,			[(offset, index, to_sql_kwargs) for offset in range(0			,			lowerCAmelCase_			,			lowerCAmelCase_		)]			,			)			,			total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size			,			unit='ba'			,			disable=not logging.is_progress_bar_enabled()			,			desc='Creating SQL from Arrow format'			,			):
														written += num_rows

								return written









"""simple docstring"""


from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


UpperCamelCase						:				Dict             =      {
    "configuration_megatron_bert": ["MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP", "MegatronBertConfig"],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	UpperCamelCase						:				Any             =      [
	    "MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST",
	    "MegatronBertForCausalLM",
	    "MegatronBertForMaskedLM",
	    "MegatronBertForMultipleChoice",
	    "MegatronBertForNextSentencePrediction",
	    "MegatronBertForPreTraining",
	    "MegatronBertForQuestionAnswering",
	    "MegatronBertForSequenceClassification",
	    "MegatronBertForTokenClassification",
	    "MegatronBertModel",
	    "MegatronBertPreTrainedModel",
	]

if TYPE_CHECKING:
	from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_megatron_bert import (
		    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MegatronBertForCausalLM,
		    MegatronBertForMaskedLM,
		    MegatronBertForMultipleChoice,
		    MegatronBertForNextSentencePrediction,
		    MegatronBertForPreTraining,
		    MegatronBertForQuestionAnswering,
		    MegatronBertForSequenceClassification,
		    MegatronBertForTokenClassification,
		    MegatronBertModel,
		    MegatronBertPreTrainedModel,
		)

else:
	import sys

	UpperCamelCase						:				Dict             =      _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





