


'''simple docstring'''



import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
		import sqlitea

		import sqlalchemy


class 		_snake_case					(     lowerCAmelCase__	):




					def __init__(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							_lowerCamelCase = False							,							**_lowerCamelCase							,							):
									super().__init__(features=_SCREAMING_SNAKE_CASE							,							cache_dir=_SCREAMING_SNAKE_CASE							,							keep_in_memory=_SCREAMING_SNAKE_CASE							,							**_SCREAMING_SNAKE_CASE)
									UpperCAmelCase__  :					Tuple        =     Sql(
									    cache_dir=_SCREAMING_SNAKE_CASE							,							features=_SCREAMING_SNAKE_CASE							,							sql=_SCREAMING_SNAKE_CASE							,							con=_SCREAMING_SNAKE_CASE							,							**_SCREAMING_SNAKE_CASE							,							)




					def 				snake_case__						(      self):
									UpperCAmelCase__  :					Optional[Any]        =     None
									UpperCAmelCase__  :					str        =     None
									UpperCAmelCase__  :					List[Any]        =     None
									UpperCAmelCase__  :					List[Any]        =     None

									self.builder.download_and_prepare(
									    download_config=_SCREAMING_SNAKE_CASE							,							download_mode=_SCREAMING_SNAKE_CASE							,							verification_mode=_SCREAMING_SNAKE_CASE							,							base_path=_SCREAMING_SNAKE_CASE							,							)

									# Build dataset for splits
									UpperCAmelCase__  :					Dict        =     self.builder.as_dataset(
									    split="""train"""							,							verification_mode=_SCREAMING_SNAKE_CASE							,							in_memory=self.keep_in_memory)
									return dataset


class 		_snake_case					:




					def __init__(      self							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							**_lowerCamelCase							,							):
									if num_proc is not None and num_proc <= 0:
													raise ValueError(f'''num_proc {num_proc} must be an integer > 0.''')

									UpperCAmelCase__  :					Optional[Any]        =     dataset
									UpperCAmelCase__  :					Any        =     name
									UpperCAmelCase__  :					Dict        =     con
									UpperCAmelCase__  :					List[str]        =     batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
									UpperCAmelCase__  :					Any        =     num_proc
									UpperCAmelCase__  :					Union[str, Any]        =     to_sql_kwargs




					def 				snake_case__						(      self):
									UpperCAmelCase__  :					str        =     self.to_sql_kwargs.pop("""sql"""							,							_SCREAMING_SNAKE_CASE)
									UpperCAmelCase__  :					Tuple        =     self.to_sql_kwargs.pop("""con"""							,							_SCREAMING_SNAKE_CASE)
									UpperCAmelCase__  :					Dict        =     self.to_sql_kwargs.pop("""index"""							,							_SCREAMING_SNAKE_CASE)

									UpperCAmelCase__  :					Tuple        =     self._write(index=_SCREAMING_SNAKE_CASE							,							**self.to_sql_kwargs)
									return written




					def 				snake_case__						(      self							,							_lowerCamelCase):
									UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					Any        =     args
									UpperCAmelCase__  :					Union[str, Any]        =     {**to_sql_kwargs, """if_exists""": """append"""} if offset > 0 else to_sql_kwargs
									UpperCAmelCase__  :					Dict        =     query_table(
									    table=self.dataset.data							,							key=slice(_SCREAMING_SNAKE_CASE							,							offset + self.batch_size)							,							indices=self.dataset._indices							,							)
									UpperCAmelCase__  :					List[str]        =     batch.to_pandas()
									UpperCAmelCase__  :					int        =     df.to_sql(self.name							,							self.con							,							index=_SCREAMING_SNAKE_CASE							,							**_SCREAMING_SNAKE_CASE)
									return num_rows or len(_SCREAMING_SNAKE_CASE)




					def 				snake_case__						(      self							,							_lowerCamelCase							,							**_lowerCamelCase):
									UpperCAmelCase__  :					Any        =     0

									if self.num_proc is None or self.num_proc == 1:
													for offset in logging.tqdm(
													    range(0							,							len(self.dataset)							,							self.batch_size)							,							unit="""ba"""							,							disable=not logging.is_progress_bar_enabled()							,							desc="""Creating SQL from Arrow format"""							,							):
																	written += self._batch_sql((offset, index, to_sql_kwargs))
									else:
													UpperCAmelCase__   ,       UpperCAmelCase__  :					str        =     len(self.dataset), self.batch_size
													with multiprocessing.Pool(self.num_proc) as pool:
																	for num_rows in logging.tqdm(
																	    pool.imap(
																	        self._batch_sql							,							[(offset, index, to_sql_kwargs) for offset in range(0							,							_SCREAMING_SNAKE_CASE							,							_SCREAMING_SNAKE_CASE)]							,							)							,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size							,							unit="""ba"""							,							disable=not logging.is_progress_bar_enabled()							,							desc="""Creating SQL from Arrow format"""							,							):
																					written += num_rows

									return written



'''simple docstring'''



import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


__A	='scheduler_config.json'


class 		_snake_case					(     a__	):
					lowerCAmelCase      :Union[str, Any]   =					1
					lowerCAmelCase      :Union[str, Any]   =					2
					lowerCAmelCase      :Optional[int]   =					3
					lowerCAmelCase      :Optional[int]   =					4
					lowerCAmelCase      :Any   =					5
					lowerCAmelCase      :Tuple   =					6
					lowerCAmelCase      :List[Any]   =					7
					lowerCAmelCase      :str   =					8
					lowerCAmelCase      :List[Any]   =					9
					lowerCAmelCase      :List[str]   =					10
					lowerCAmelCase      :Union[str, Any]   =					11
					lowerCAmelCase      :Optional[int]   =					12
					lowerCAmelCase      :str   =					13
					lowerCAmelCase      :Dict   =					14


@dataclass
class 		_snake_case					(     a__	):
					lowerCAmelCase      :torch.FloatTensor


class 		_snake_case					:
					lowerCAmelCase      :str   =					SCHEDULER_CONFIG_NAME
					lowerCAmelCase      :Union[str, Any]   =					[]
					lowerCAmelCase      :List[str]   =					True




					@classmethod
					def 				snake_case__						(      cls							,							_lowerCamelCase = None							,							_lowerCamelCase = None							,							_lowerCamelCase=False							,							**_lowerCamelCase							,							):
								UpperCAmelCase__   ,       UpperCAmelCase__   ,       UpperCAmelCase__  :					Optional[int]        =     cls.load_config(
								    pretrained_model_name_or_path=_lowerCamelCase							,							subfolder=_lowerCamelCase							,							return_unused_kwargs=_lowerCamelCase							,							return_commit_hash=_lowerCamelCase							,							**_lowerCamelCase							,							)
								return cls.from_config(_lowerCamelCase							,							return_unused_kwargs=_lowerCamelCase							,							**_lowerCamelCase)




					def 				snake_case__						(      self							,							_lowerCamelCase							,							_lowerCamelCase = False							,							**_lowerCamelCase):
								self.save_config(save_directory=_lowerCamelCase							,							push_to_hub=_lowerCamelCase							,							**_lowerCamelCase)




					@property
					def 				snake_case__						(      self):
								return self._get_compatibles()




					@classmethod
					def 				snake_case__						(      cls):
								UpperCAmelCase__  :					List[Any]        =     list(set([cls.__name__] + cls._compatibles))
								UpperCAmelCase__  :					int        =     importlib.import_module(__name__.split(""".""")[0])
								UpperCAmelCase__  :					Dict        =     [
								    getattr(_lowerCamelCase							,							_lowerCamelCase) for c in compatible_classes_str if hasattr(_lowerCamelCase							,							_lowerCamelCase)
								]
								return compatible_classes