






import os
from typing import BinaryIO, Optional, Union

import numpy as np
import pyarrow.parquet as pq

from .. import Audio, Dataset, Features, Image, NamedSplit, Value, config
from ..features.features import FeatureType, _visit
from ..formatting import query_table
from ..packaged_modules import _PACKAGED_DATASETS_MODULES
from ..packaged_modules.parquet.parquet import Parquet
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader







def     a_   (__A      )				->						Optional[int]:



					"""simple docstring"""
					__a					:						List[Any]      =					np.inf

					def set_batch_size(__A      ) -> None:
										nonlocal batch_size
										if isinstance(__A				,       __A      ):
															__a					:						Optional[Any]      =					min(__A				,       config.PARQUET_ROW_GROUP_SIZE_FOR_IMAGE_DATASETS      )
										elif isinstance(__A				,       __A      ):
															__a					:						int      =					min(__A				,       config.PARQUET_ROW_GROUP_SIZE_FOR_AUDIO_DATASETS      )
										elif isinstance(__A				,       __A      ) and feature.dtype == "binary":
															__a					:						List[Any]      =					min(__A				,       config.PARQUET_ROW_GROUP_SIZE_FOR_BINARY_DATASETS      )

					_visit(__A				,       __A      )

					return None if batch_size is np.inf else batch_size




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						def __init__(self:					str ,					__UpperCAmelCase:					NestedDataStructureLike[PathLike] ,					__UpperCAmelCase:					Optional[NamedSplit] = None ,					__UpperCAmelCase:					Optional[Features] = None ,					__UpperCAmelCase:					str = None ,					__UpperCAmelCase:					bool = False ,					__UpperCAmelCase:					bool = False ,					__UpperCAmelCase:					Optional[int] = None ,					**__UpperCAmelCase:					Tuple ,					)    ->     List[str]:




											'''simple docstring'''



											super().__init__(
											    __UpperCAmelCase ,					split=__UpperCAmelCase ,					features=__UpperCAmelCase ,					cache_dir=__UpperCAmelCase ,					keep_in_memory=__UpperCAmelCase ,					streaming=__UpperCAmelCase ,					num_proc=__UpperCAmelCase ,					**__UpperCAmelCase ,					)
											__a					:						Tuple      =					path_or_paths if isinstance(__UpperCAmelCase ,					__UpperCAmelCase   ) else {self.split: path_or_paths}
											__a					:						Optional[Any]      =					_PACKAGED_DATASETS_MODULES["parquet"][1]
											__a					:						List[str]      =					Parquet(
											    cache_dir=__UpperCAmelCase ,					data_files=__UpperCAmelCase ,					features=__UpperCAmelCase ,					hash=__UpperCAmelCase ,					**__UpperCAmelCase ,					)



						def        UpperCAmelCase__					(self:					Dict   )    ->     List[str]:




											'''simple docstring'''



											if self.streaming:
																__a					:						Union[str, Any]      =					self.builder.as_streaming_dataset(split=self.split   )
											# Build regular (map-style) dataset
											else:
																__a					:						Optional[Any]      =					None
																__a					:						Union[str, Any]      =					None
																__a					:						Tuple      =					None
																__a					:						Any      =					None

																self.builder.download_and_prepare(
																    download_config=__UpperCAmelCase ,					download_mode=__UpperCAmelCase ,					verification_mode=__UpperCAmelCase ,					base_path=__UpperCAmelCase ,					num_proc=self.num_proc ,					)
																__a					:						Any      =					self.builder.as_dataset(
																    split=self.split ,					verification_mode=__UpperCAmelCase ,					in_memory=self.keep_in_memory   )
											return dataset




class 							snake_case_       :

						"""simple docstring"""



						def __init__(self:					List[str] ,					__UpperCAmelCase:					Dataset ,					__UpperCAmelCase:					Union[PathLike, BinaryIO] ,					__UpperCAmelCase:					Optional[int] = None ,					**__UpperCAmelCase:					Optional[Any] ,					)    ->     str:




											'''simple docstring'''



											__a					:						Union[str, Any]      =					dataset
											__a					:						Optional[Any]      =					path_or_buf
											__a					:						Optional[Any]      =					batch_size or get_writer_batch_size(dataset.features   )
											__a					:						str      =					parquet_writer_kwargs



						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     int:




											'''simple docstring'''



											__a					:						int      =					self.batch_size if self.batch_size else config.DEFAULT_MAX_BATCH_SIZE

											if isinstance(self.path_or_buf ,					(str, bytes, os.PathLike)   ):
																with open(self.path_or_buf ,					"wb+"   ) as buffer:
																					__a					:						Optional[int]      =					self._write(file_obj=__UpperCAmelCase ,					batch_size=__UpperCAmelCase ,					**self.parquet_writer_kwargs   )
											else:
																__a					:						str      =					self._write(file_obj=self.path_or_buf ,					batch_size=__UpperCAmelCase ,					**self.parquet_writer_kwargs   )
											return written



						def        UpperCAmelCase__					(self:					str ,					__UpperCAmelCase:					BinaryIO ,					__UpperCAmelCase:					int ,					**__UpperCAmelCase:					str   )    ->     int:




											'''simple docstring'''



											__a					:						str      =					0
											__a					:						Tuple      =					parquet_writer_kwargs.pop("path_or_buf" ,					__UpperCAmelCase   )
											__a					:						Any      =					self.dataset.features.arrow_schema

											__a					:						List[str]      =					pq.ParquetWriter(__UpperCAmelCase ,					schema=__UpperCAmelCase ,					**__UpperCAmelCase   )

											for offset in logging.tqdm(
											    range(0 ,					len(self.dataset   ) ,					__UpperCAmelCase   ) ,					unit="ba" ,					disable=not logging.is_progress_bar_enabled() ,					desc="Creating parquet from Arrow format" ,					):
																__a					:						Optional[Any]      =					query_table(
																    table=self.dataset._data ,					key=slice(__UpperCAmelCase ,					offset + batch_size   ) ,					indices=self.dataset._indices if self.dataset._indices is not None else None ,					)
																writer.write_table(__UpperCAmelCase   )
																written += batch.nbytes
											writer.close()
											return written













import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
			import torch_xla.core.xla_model as xm
			import torch_xla.debug.metrics as met




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						def __init__(self:					str ,					*__UpperCAmelCase:					Optional[int] ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Dict=None ,					**__UpperCAmelCase:					Tuple   )    ->     Dict:




											'''simple docstring'''



											super().__init__(*__UpperCAmelCase ,					**__UpperCAmelCase   )
											__a					:						str      =					eval_examples
											__a					:						List[Any]      =					post_process_function



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					Optional[Dataset] = None ,					__UpperCAmelCase:					Optional[int]=None ,					__UpperCAmelCase:					Optional[List[str]] = None ,					__UpperCAmelCase:					str = "eval" ,					**__UpperCAmelCase:					Any ,					)    ->     Dict[str, float]:




											'''simple docstring'''



											__a					:						Optional[int]      =					gen_kwargs.copy()
											__a					:						Union[str, Any]      =					(
											    gen_kwargs["max_length"] if gen_kwargs.get("max_length"   ) is not None else self.args.generation_max_length
											)
											__a					:						Dict      =					(
											    gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"   ) is not None else self.args.generation_num_beams
											)
											__a					:						Optional[Any]      =					gen_kwargs

											__a					:						Union[str, Any]      =					self.eval_dataset if eval_dataset is None else eval_dataset
											__a					:						Optional[int]      =					self.get_eval_dataloader(__UpperCAmelCase   )
											__a					:						Tuple      =					self.eval_examples if eval_examples is None else eval_examples

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						int      =					self.compute_metrics
											__a					:						Dict      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						str      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						str      =					eval_loop(
																    __UpperCAmelCase ,					description="Evaluation" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						Union[str, Any]      =					compute_metrics
											__a					:						str      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )

											if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																# Only the main node write the results by default
																__a					:						Union[str, Any]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase   )
																__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

																# Prefix all keys with metric_key_prefix + '_'
																for key in list(metrics.keys()   ):
																					if not key.startswith(f'{metric_key_prefix}_'   ):
																										__a					:						int      =					metrics.pop(__UpperCAmelCase   )

																metrics.update(output.metrics   )
											else:
																__a					:						Optional[int]      =					output.metrics

											if self.args.should_log:
																# Only the main node log the results by default
																self.log(__UpperCAmelCase   )

											if self.args.tpu_metrics_debug or self.args.debug:
																# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																xm.master_print(met.metrics_report()   )

											__a					:						Tuple      =					self.callback_handler.on_evaluate(self.args ,					self.state ,					self.control ,					__UpperCAmelCase   )
											return metrics



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					Union[str, Any]=None ,					__UpperCAmelCase:					str = "test" ,					**__UpperCAmelCase:					List[Any]   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						Any      =					gen_kwargs.copy()

											__a					:						Tuple      =					self.get_test_dataloader(__UpperCAmelCase   )

											# Temporarily disable metric computation, we will do it in the loop here.
											__a					:						Optional[Any]      =					self.compute_metrics
											__a					:						int      =					None
											__a					:						List[Any]      =					time.time()
											__a					:						Dict      =					self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
											try:
																__a					:						Optional[Any]      =					eval_loop(
																    __UpperCAmelCase ,					description="Prediction" ,					prediction_loss_only=True if compute_metrics is None else None ,					ignore_keys=__UpperCAmelCase ,					metric_key_prefix=__UpperCAmelCase ,					)
											finally:
																__a					:						List[Any]      =					compute_metrics

											__a					:						Optional[Any]      =					self.args.eval_batch_size * self.args.world_size
											if f'{metric_key_prefix}_jit_compilation_time' in output.metrics:
																start_time += output.metrics[f'{metric_key_prefix}_jit_compilation_time']
											output.metrics.update(
											    speed_metrics(
											        __UpperCAmelCase ,					__UpperCAmelCase ,					num_samples=output.num_samples ,					num_steps=math.ceil(output.num_samples / total_batch_size   ) ,					)   )
											if self.post_process_function is None or self.compute_metrics is None:
																return output

											__a					:						Optional[int]      =					self.post_process_function(__UpperCAmelCase ,					__UpperCAmelCase ,					__UpperCAmelCase ,					"predict"   )
											__a					:						Dict      =					self.compute_metrics(__UpperCAmelCase   )

											# Prefix all keys with metric_key_prefix + '_'
											for key in list(metrics.keys()   ):
																if not key.startswith(f'{metric_key_prefix}_'   ):
																					__a					:						List[str]      =					metrics.pop(__UpperCAmelCase   )
											metrics.update(output.metrics   )
											return PredictionOutput(predictions=predictions.predictions ,					label_ids=predictions.label_ids ,					metrics=__UpperCAmelCase   )





