
'''simple docstring'''







import multiprocessing
from typing import TYPE_CHECKING, Optional, Union

from .. import Dataset, Features, config
from ..formatting import query_table
from ..packaged_modules.sql.sql import Sql
from ..utils import logging
from .abc import AbstractDatasetInputStream


if TYPE_CHECKING:
							import sqlitea

							import sqlalchemy






class 			lowerCAmelCase__       (  UpperCamelCase_					):






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = False    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(features=_a    ,							cache_dir=_a    ,							keep_in_memory=_a    ,							**_a		)
											lowercase_     :					int													=	Sql(
											    cache_dir=_a    ,							features=_a    ,							sql=_a    ,							con=_a    ,							**_a    ,							)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	None
											lowercase_     :					List[Any]													=	None
											lowercase_     :					Any													=	None
											lowercase_     :					Tuple													=	None

											self.builder.download_and_prepare(
											    download_config=_a    ,							download_mode=_a    ,							verification_mode=_a    ,							base_path=_a    ,							)

											# Build dataset for splits
											lowercase_     :					int													=	self.builder.as_dataset(
											    split='''train'''    ,							verification_mode=_a    ,							in_memory=self.keep_in_memory		)
											return dataset







class 			lowerCAmelCase__       :






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None    ,							__SCREAMING_SNAKE_CASE = None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											if num_proc is not None and num_proc <= 0:
																	raise ValueError(F'''num_proc {num_proc} must be an integer > 0.'''		)

											lowercase_     :					str													=	dataset
											lowercase_     :					Any													=	name
											lowercase_     :					Dict													=	con
											lowercase_     :					Dict													=	batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
											lowercase_     :					List[str]													=	num_proc
											lowercase_     :					Union[str, Any]													=	to_sql_kwargs






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	self.to_sql_kwargs.pop('''sql'''    ,							_a		)
											lowercase_     :					Optional[Any]													=	self.to_sql_kwargs.pop('''con'''    ,							_a		)
											lowercase_     :					List[str]													=	self.to_sql_kwargs.pop('''index'''    ,							_a		)

											lowercase_     :					Optional[int]													=	self._write(index=_a    ,							**self.to_sql_kwargs		)
											return written






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Any													=	args
											lowercase_     :					List[Any]													=	{**to_sql_kwargs, """if_exists""": """append"""} if offset > 0 else to_sql_kwargs
											lowercase_     :					Union[str, Any]													=	query_table(
											    table=self.dataset.data    ,							key=slice(_a    ,							offset + self.batch_size		)    ,							indices=self.dataset._indices    ,							)
											lowercase_     :					Any													=	batch.to_pandas()
											lowercase_     :					Any													=	df.to_sql(self.name    ,							self.con    ,							index=_a    ,							**_a		)
											return num_rows or len(_a		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[int]													=	0

											if self.num_proc is None or self.num_proc == 1:
																	for offset in logging.tqdm(
																	    range(0    ,							len(self.dataset		)    ,							self.batch_size		)    ,							unit='''ba'''    ,							disable=not logging.is_progress_bar_enabled()    ,							desc='''Creating SQL from Arrow format'''    ,							):
																							written += self._batch_sql((offset, index, to_sql_kwargs)		)
											else:
																	lowercase_     :					Any													=	len(self.dataset		), self.batch_size
																	with multiprocessing.Pool(self.num_proc		) as pool:
																							for num_rows in logging.tqdm(
																							    pool.imap(
																							        self._batch_sql    ,							[(offset, index, to_sql_kwargs) for offset in range(0    ,							_a    ,							_a		)]    ,							)    ,							total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size    ,							unit='''ba'''    ,							disable=not logging.is_progress_bar_enabled()    ,							desc='''Creating SQL from Arrow format'''    ,							):
																													written += num_rows

											return written


'''simple docstring'''







import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


_lowercase			:  Tuple   							=	logging.getLogger(__name__)






@dataclass
@add_start_docstrings(TrainingArguments.__doc__					)
class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					field(default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use SortishSampler or not.'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `max_length` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `num_beams` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': '''Model id, file path or url pointing to a GenerationConfig json file, to use during prediction.'''
					    }   ,						)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	super().to_dict()
											for k, v in d.items():
																	if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							lowercase_     :					str													=	v.to_dict()
											return d



