
'''simple docstring'''







import itertools
from dataclasses import dataclass
from typing import Optional

import pandas as pd
import pyarrow as pa

import datasets
from datasets.table import table_cast






@dataclass
class 			lowerCAmelCase__       (  datasets.BuilderConfig					):
					lowerCAmelCase_					       =					None







class 			lowerCAmelCase__       (  datasets.ArrowBasedBuilder					):
					lowerCAmelCase_					       =					PandasConfig






					def      _snake_case   (       self		):


											"""simple docstring"""





											return datasets.DatasetInfo(features=self.config.features		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if not self.config.data_files:
																	raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''		)
											lowercase_     :					List[Any]													=	dl_manager.download_and_extract(self.config.data_files		)
											if isinstance(__SCREAMING_SNAKE_CASE    ,							(str, list, tuple)		):
																	lowercase_     :					Optional[Any]													=	data_files
																	if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							lowercase_     :					Optional[Any]													=	[files]
																	# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																	lowercase_     :					Any													=	[dl_manager.iter_files(__SCREAMING_SNAKE_CASE		) for file in files]
																	return [datasets.SplitGenerator(name=datasets.Split.TRAIN    ,							gen_kwargs={'''files''': files}		)]
											lowercase_     :					Dict													=	[]
											for split_name, files in data_files.items():
																	if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							lowercase_     :					Optional[Any]													=	[files]
																	# Use `dl_manager.iter_files` to skip hidden files in an extracted archive
																	lowercase_     :					List[str]													=	[dl_manager.iter_files(__SCREAMING_SNAKE_CASE		) for file in files]
																	splits.append(datasets.SplitGenerator(name=__SCREAMING_SNAKE_CASE    ,							gen_kwargs={'''files''': files}		)		)
											return splits






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if self.config.features is not None:
																	# more expensive cast to support nested features with keys in a different order
																	# allows str <-> int/float or str to Audio for example
																	lowercase_     :					str													=	table_cast(__SCREAMING_SNAKE_CASE    ,							self.config.features.arrow_schema		)
											return pa_table






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											for i, file in enumerate(itertools.chain.from_iterable(__SCREAMING_SNAKE_CASE		)		):
																	with open(__SCREAMING_SNAKE_CASE    ,							'''rb'''		) as f:
																							lowercase_     :					Dict													=	pa.Table.from_pandas(pd.read_pickle(__SCREAMING_SNAKE_CASE		)		)
																							yield i, self._cast_table(__SCREAMING_SNAKE_CASE		)


'''simple docstring'''







import logging
from dataclasses import dataclass, field
from pathlib import Path
from typing import Optional, Union

from .generation.configuration_utils import GenerationConfig
from .training_args import TrainingArguments
from .utils import add_start_docstrings


_lowercase			:  Tuple   							=	logging.getLogger(__name__)






@dataclass
@add_start_docstrings(TrainingArguments.__doc__					)
class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					field(default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use SortishSampler or not.'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={'''help''': '''Whether to use generate to calculate generative metrics (ROUGE, BLEU).'''}					)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `max_length` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `max_length` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': (
					            '''The `num_beams` to use on each evaluation loop when `predict_with_generate=True`. Will default '''
					            '''to the `num_beams` value of the model configuration.'''
					        )
					    }   ,						)
					lowerCAmelCase_					       =					field(
					    default=lowerCamelCase_   ,						metadata={
					        '''help''': '''Model id, file path or url pointing to a GenerationConfig json file, to use during prediction.'''
					    }   ,						)






					def      _snake_case   (       self		):


											"""simple docstring"""





											lowercase_     :					Dict													=	super().to_dict()
											for k, v in d.items():
																	if isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		):
																							lowercase_     :					str													=	v.to_dict()
											return d



