






"""simple docstring"""
import math_equivalence  # From: git+https://github.com/hendrycks/math.git

import datasets


lowerCamelCase_							      = '''\
@article{hendrycksmath2021,
  title={Measuring Mathematical Problem Solving With the MATH Dataset},
  author={Dan Hendrycks
    and Collin Burns
    and Saurav Kadavath
    and Akul Arora
    and Steven Basart
    and Eric Tang
    and Dawn Song
    and Jacob Steinhardt},
  journal={arXiv preprint arXiv:2103.03874},
  year={2021}
}
'''


lowerCamelCase_							      = '''\
This metric is used to assess performance on the Mathematics Aptitude Test of Heuristics (MATH) dataset.
It first canonicalizes the inputs (e.g., converting "1/2" to "\\frac{1}{2}") and then computes accuracy.
'''


lowerCamelCase_							      = r'''
Calculates accuracy after canonicalizing inputs.

Args:
    predictions: list of predictions to score. Each prediction
        is a string that contains natural language and LaTex.
    references: list of reference for each prediction. Each
        reference is a string that contains natural language
        and LaTex.
Returns:
    accuracy: accuracy after canonicalizing inputs
        (e.g., converting "1/2" to "\\frac{1}{2}")

Examples:
    >>> metric = datasets.load_metric("competition_math")
    >>> results = metric.compute(references=["\\frac{1}{2}"], predictions=["1/2"])
    >>> print(results)
    {\'accuracy\': 1.0}
'''







@datasets.utils.file_utils.add_end_docstrings(_DESCRIPTION							,		_KWARGS_DESCRIPTION	)
class      UpperCamelCase_			(datasets.Metric	):





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]		)		->     Optional[int]:
				return datasets.MetricInfo(
				    description=_DESCRIPTION      ,      citation=_CITATION      ,      inputs_description=_KWARGS_DESCRIPTION      ,      features=datasets.Features(
				        {
				            "predictions": datasets.Value("string"		),
				            "references": datasets.Value("string"		),
				        }		)      ,      homepage="https://github.com/hendrycks/math"      ,      codebase_urls=["https://github.com/hendrycks/math"]      ,      )





	def 	_SCREAMING_SNAKE_CASE ( self   :					List[str]      ,      lowerCAmelCase_   :					Optional[int]      ,      lowerCAmelCase_   :					Union[str, Any]		)		->     Optional[Any]:
				UpperCAmelCase_	:					Optional[Any]     	=   0.0
				for i, j in zip(lowerCAmelCase_      ,      lowerCAmelCase_		):
							n_correct += 1.0 if math_equivalence.is_equiv(lowerCAmelCase_      ,      lowerCAmelCase_		) else 0.0
				UpperCAmelCase_	:					Any     	=   n_correct / len(lowerCAmelCase_		)
				return {
				    "accuracy": accuracy,
				}







'''simple docstring'''






import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
 import torch_xla.core.xla_model as xm
 import torch_xla.debug.metrics as met






class    __magic_name__				(						lowerCAmelCase							):






       def __init__(     self    ,				*snake_case    ,				snake_case=None    ,				snake_case=None    ,				**snake_case)		->    Tuple:




          '''simple docstring'''
          super().__init__(*snake_case    ,				**snake_case)
          _UpperCAmelCase  :  Any				       =eval_examples
          _UpperCAmelCase  :  Optional[int]				       =post_process_function






       def      lowerCAmelCase						(     self    ,				snake_case = None    ,				snake_case=None    ,				snake_case = None    ,				snake_case = "eval"    ,				**snake_case    ,				)		->    Dict[str, float]:




          '''simple docstring'''
          _UpperCAmelCase  :  Tuple				       =gen_kwargs.copy()
          _UpperCAmelCase  :  int				       =(
              gen_kwargs['max_length'] if gen_kwargs.get('max_length') is not None else self.args.generation_max_length
          )
          _UpperCAmelCase  :  Dict				       =(
              gen_kwargs['num_beams'] if gen_kwargs.get('num_beams') is not None else self.args.generation_num_beams
          )
          _UpperCAmelCase  :  Dict				       =gen_kwargs

          _UpperCAmelCase  :  Tuple				       =self.eval_dataset if eval_dataset is None else eval_dataset
          _UpperCAmelCase  :  Union[str, Any]				       =self.get_eval_dataloader(snake_case)
          _UpperCAmelCase  :  Tuple				       =self.eval_examples if eval_examples is None else eval_examples

          # Temporarily disable metric computation, we will do it in the loop here.
          _UpperCAmelCase  :  Dict				       =self.compute_metrics
          _UpperCAmelCase  :  Optional[Any]				       =None
          _UpperCAmelCase  :  str				       =time.time()
          _UpperCAmelCase  :  int				       =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _UpperCAmelCase  :  str				       =eval_loop(
                 snake_case    ,				description='Evaluation'    ,				prediction_loss_only=True if compute_metrics is None else None    ,				ignore_keys=snake_case    ,				metric_key_prefix=snake_case    ,				)
          finally:
             _UpperCAmelCase  :  str				       =compute_metrics
          _UpperCAmelCase  :  Optional[int]				       =self.args.eval_batch_size * self.args.world_size
          if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
             start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
          output.metrics.update(
              speed_metrics(
                  snake_case    ,				snake_case    ,				num_samples=output.num_samples    ,				num_steps=math.ceil(output.num_samples / total_batch_size)    ,				))

          if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
             # Only the main node write the results by default
             _UpperCAmelCase  :  Tuple				       =self.post_process_function(snake_case    ,				snake_case    ,				snake_case)
             _UpperCAmelCase  :  Union[str, Any]				       =self.compute_metrics(snake_case)

             # Prefix all keys with metric_key_prefix + '_'
             for key in list(metrics.keys()):
                if not key.startswith(f"{metric_key_prefix}_"):
                   _UpperCAmelCase  :  Any				       =metrics.pop(snake_case)

             metrics.update(output.metrics)
          else:
             _UpperCAmelCase  :  str				       =output.metrics

          if self.args.should_log:
             # Only the main node log the results by default
             self.log(snake_case)

          if self.args.tpu_metrics_debug or self.args.debug:
             # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
             xm.master_print(met.metrics_report())

          _UpperCAmelCase  :  Any				       =self.callback_handler.on_evaluate(self.args    ,				self.state    ,				self.control    ,				snake_case)
          return metrics






       def      lowerCAmelCase						(     self    ,				snake_case    ,				snake_case    ,				snake_case=None    ,				snake_case = "test"    ,				**snake_case)		->    Dict:




          '''simple docstring'''
          _UpperCAmelCase  :  Optional[int]				       =gen_kwargs.copy()

          _UpperCAmelCase  :  int				       =self.get_test_dataloader(snake_case)

          # Temporarily disable metric computation, we will do it in the loop here.
          _UpperCAmelCase  :  str				       =self.compute_metrics
          _UpperCAmelCase  :  Union[str, Any]				       =None
          _UpperCAmelCase  :  Union[str, Any]				       =time.time()
          _UpperCAmelCase  :  Tuple				       =self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
          try:
             _UpperCAmelCase  :  Any				       =eval_loop(
                 snake_case    ,				description='Prediction'    ,				prediction_loss_only=True if compute_metrics is None else None    ,				ignore_keys=snake_case    ,				metric_key_prefix=snake_case    ,				)
          finally:
             _UpperCAmelCase  :  Any				       =compute_metrics

          _UpperCAmelCase  :  Dict				       =self.args.eval_batch_size * self.args.world_size
          if f"{metric_key_prefix}_jit_compilation_time" in output.metrics:
             start_time += output.metrics[f"{metric_key_prefix}_jit_compilation_time"]
          output.metrics.update(
              speed_metrics(
                  snake_case    ,				snake_case    ,				num_samples=output.num_samples    ,				num_steps=math.ceil(output.num_samples / total_batch_size)    ,				))
          if self.post_process_function is None or self.compute_metrics is None:
             return output

          _UpperCAmelCase  :  Tuple				       =self.post_process_function(snake_case    ,				snake_case    ,				snake_case    ,				'predict')
          _UpperCAmelCase  :  Union[str, Any]				       =self.compute_metrics(snake_case)

          # Prefix all keys with metric_key_prefix + '_'
          for key in list(metrics.keys()):
             if not key.startswith(f"{metric_key_prefix}_"):
                _UpperCAmelCase  :  Any				       =metrics.pop(snake_case)
          metrics.update(output.metrics)
          return PredictionOutput(predictions=predictions.predictions    ,				label_ids=predictions.label_ids    ,				metrics=snake_case)

