


'''simple docstring'''






import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
		import torch_xla.core.xla_model as xm
		import torch_xla.debug.metrics as met




class 						a__							(				a__    ):





							'''simple docstring'''






							def __init__(						self   ,       *lowerCamelCase_   ,       lowerCamelCase_=None   ,       lowerCamelCase_=None   ,       **lowerCamelCase_  )			->				List[Any]:
														super().__init__(*lowerCamelCase_   ,       **lowerCamelCase_  )
														lowerCAmelCase__			  =      eval_examples
														lowerCAmelCase__			  =      post_process_function







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_ = None   ,       lowerCamelCase_=None   ,       lowerCamelCase_ = None   ,       lowerCamelCase_ = "eval"   ,       **lowerCamelCase_   ,       )			->				Dict[str, float]:
														lowerCAmelCase__			  =      gen_kwargs.copy()
														lowerCAmelCase__			  =      (
														    gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length'''  ) is not None else self.args.generation_max_length
														)
														lowerCAmelCase__			  =      (
														    gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams'''  ) is not None else self.args.generation_num_beams
														)
														lowerCAmelCase__			  =      gen_kwargs

														lowerCAmelCase__			  =      self.eval_dataset if eval_dataset is None else eval_dataset
														lowerCAmelCase__			  =      self.get_eval_dataloader(lowerCamelCase_  )
														lowerCAmelCase__			  =      self.eval_examples if eval_examples is None else eval_examples

														# Temporarily disable metric computation, we will do it in the loop here.
														lowerCAmelCase__			  =      self.compute_metrics
														lowerCAmelCase__			  =      None
														lowerCAmelCase__			  =      time.time()
														lowerCAmelCase__			  =      self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
														try:
																					lowerCAmelCase__			  =      eval_loop(
																					    lowerCamelCase_   ,       description='''Evaluation'''   ,       prediction_loss_only=True if compute_metrics is None else None   ,       ignore_keys=lowerCamelCase_   ,       metric_key_prefix=lowerCamelCase_   ,       )
														finally:
																					lowerCAmelCase__			  =      compute_metrics
														lowerCAmelCase__			  =      self.args.eval_batch_size * self.args.world_size
														if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																					start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
														output.metrics.update(
														    speed_metrics(
														        lowerCamelCase_   ,       lowerCamelCase_   ,       num_samples=output.num_samples   ,       num_steps=math.ceil(output.num_samples / total_batch_size  )   ,       )  )

														if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
																					# Only the main node write the results by default
																					lowerCAmelCase__			  =      self.post_process_function(lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_  )
																					lowerCAmelCase__			  =      self.compute_metrics(lowerCamelCase_  )

																					# Prefix all keys with metric_key_prefix + '_'
																					for key in list(metrics.keys()  ):
																												if not key.startswith(F"""{metric_key_prefix}_"""  ):
																																			lowerCAmelCase__			  =      metrics.pop(lowerCamelCase_  )

																					metrics.update(output.metrics  )
														else:
																					lowerCAmelCase__			  =      output.metrics

														if self.args.should_log:
																					# Only the main node log the results by default
																					self.log(lowerCamelCase_  )

														if self.args.tpu_metrics_debug or self.args.debug:
																					# tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
																					xm.master_print(met.metrics_report()  )

														lowerCAmelCase__			  =      self.callback_handler.on_evaluate(self.args   ,       self.state   ,       self.control   ,       lowerCamelCase_  )
														return metrics







							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_=None   ,       lowerCamelCase_ = "test"   ,       **lowerCamelCase_  )			->				int:
														lowerCAmelCase__			  =      gen_kwargs.copy()

														lowerCAmelCase__			  =      self.get_test_dataloader(lowerCamelCase_  )

														# Temporarily disable metric computation, we will do it in the loop here.
														lowerCAmelCase__			  =      self.compute_metrics
														lowerCAmelCase__			  =      None
														lowerCAmelCase__			  =      time.time()
														lowerCAmelCase__			  =      self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
														try:
																					lowerCAmelCase__			  =      eval_loop(
																					    lowerCamelCase_   ,       description='''Prediction'''   ,       prediction_loss_only=True if compute_metrics is None else None   ,       ignore_keys=lowerCamelCase_   ,       metric_key_prefix=lowerCamelCase_   ,       )
														finally:
																					lowerCAmelCase__			  =      compute_metrics

														lowerCAmelCase__			  =      self.args.eval_batch_size * self.args.world_size
														if F"""{metric_key_prefix}_jit_compilation_time""" in output.metrics:
																					start_time += output.metrics[F"""{metric_key_prefix}_jit_compilation_time"""]
														output.metrics.update(
														    speed_metrics(
														        lowerCamelCase_   ,       lowerCamelCase_   ,       num_samples=output.num_samples   ,       num_steps=math.ceil(output.num_samples / total_batch_size  )   ,       )  )
														if self.post_process_function is None or self.compute_metrics is None:
																					return output

														lowerCAmelCase__			  =      self.post_process_function(lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_   ,       '''predict'''  )
														lowerCAmelCase__			  =      self.compute_metrics(lowerCamelCase_  )

														# Prefix all keys with metric_key_prefix + '_'
														for key in list(metrics.keys()  ):
																					if not key.startswith(F"""{metric_key_prefix}_"""  ):
																												lowerCAmelCase__			  =      metrics.pop(lowerCamelCase_  )
														metrics.update(output.metrics  )
														return PredictionOutput(predictions=predictions.predictions   ,       label_ids=predictions.label_ids   ,       metrics=lowerCamelCase_  )



'''simple docstring'''






# Copyright 2021 The HuggingFace Team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import argparse

from ...utils.dataclasses import (
    ComputeEnvironment,
    DistributedType,
    DynamoBackend,
    PrecisionType,
    SageMakerDistributedType,
)
from ..menu import BulletMenu


__UpperCAmelCase       							=				[
    '''EAGER''',
    '''AOT_EAGER''',
    '''INDUCTOR''',
    '''NVFUSER''',
    '''AOT_NVFUSER''',
    '''AOT_CUDAGRAPHS''',
    '''OFI''',
    '''FX2TRT''',
    '''ONNXRT''',
    '''IPEX''',
]


def    _snake_case  (	A					,       A=None					,       A=None					,       A=None			)		->       Union[str, Any]:
							lowerCAmelCase__			  =      True
							while ask_again:
														lowerCAmelCase__			  =      input(A			)
														try:
																					if default is not None and len(A			) == 0:
																												return default
																					return convert_value(A			) if convert_value is not None else result
														except Exception:
																					if error_message is not None:
																												print(A			)


def    _snake_case  (	A					,       A=[]					,       A=None					,       A=0			)		->       List[Any]:
							lowerCAmelCase__			  =      BulletMenu(A					,       A			)
							lowerCAmelCase__			  =      menu.run(default_choice=A			)
							return convert_value(A			) if convert_value is not None else result


def    _snake_case  (	A			)		->       Tuple:
							lowerCAmelCase__			  =      int(A			)
							return ComputeEnvironment(['''LOCAL_MACHINE''', '''AMAZON_SAGEMAKER'''][value]			)


def    _snake_case  (	A			)		->       Union[str, Any]:
							lowerCAmelCase__			  =      int(A			)
							return DistributedType(['''NO''', '''MULTI_CPU''', '''MULTI_XPU''', '''MULTI_GPU''', '''MULTI_NPU''', '''TPU'''][value]			)


def    _snake_case  (	A			)		->       str:
							lowerCAmelCase__			  =      int(A			)
							return DynamoBackend(DYNAMO_BACKENDS[value]			).value


def    _snake_case  (	A			)		->       Tuple:
							lowerCAmelCase__			  =      int(A			)
							return PrecisionType(['''no''', '''fp16''', '''bf16''', '''fp8'''][value]			)


def    _snake_case  (	A			)		->       Union[str, Any]:
							lowerCAmelCase__			  =      int(A			)
							return SageMakerDistributedType(['''NO''', '''DATA_PARALLEL''', '''MODEL_PARALLEL'''][value]			)


def    _snake_case  (	A			)		->       List[str]:
							return {"yes": True, "no": False}[value.lower()]






class 						a__							(				argparse.RawDescriptionHelpFormatter    ):





							'''simple docstring'''






							def     __SCREAMING_SNAKE_CASE				(						self   ,       lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_  )			->				Dict:
														lowerCAmelCase__			  =      super()._format_usage(lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_   ,       lowerCamelCase_  )
														lowerCAmelCase__			  =      usage.replace('''<command> [<args>] '''   ,       ''''''  )
														return usage