class_docstr: The MATH dataset from the paper
  "Measuring Mathematical Problem Solving With the MATH Dataset"
  by Hendrycks et al. (2021):
  https://arxiv.org/pdf/2103.03874.pdf
  
  Example input, using official examples:
  
  ```
  Given a mathematics problem, determine the answer. Simplify your answer as much as possible.
  ###
  Problem: What is $\left(rac{7}{8}
  ight)^3 \cdot \left(rac{7}{8}
  ight)^{-3}$?
  Answer: $1$
  ###
  Problem: In how many ways can 4 books be selected from a shelf of 6 books if the order in which the books are selected does not matter?
  Answer: $15$
  ###
  Problem: Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$
  Answer: $\sqrt{59}$
  ###
  Problem: The faces of an octahedral die are labeled with digits $1$ through $8$. What is the probability, expressed as a common fraction, of rolling a sum of $15$ with a pair of such octahedral dice?
  Answer: $rac{1}{32}$
  ###
  Problem: The first three terms of an arithmetic sequence are 1, 10 and 19, respectively. What is the value of the 21st term?
  Answer: $181$
  ###
  Problem: Calculate $6 \cdot 8rac{1}{3}
  Answer: $50$
  ###
  Problem: When the binary number $100101110010_2$ is divided by 4, what is the remainder (give your answer in base 10)?
  Answer: $2$
  ###
  Problem: How many zeros are at the end of the product 25 $  imes$ 240?
  Answer: $3$
  ###
  Problem: What is $\dbinom{n}{n}$ for any positive integer $n$?
  Answer: $
  ```
  
  Example expected output
  
  ```
  1$
  ```
class_docstr_tok: ["The","MATH","dataset","from","the","paper","''","Measuring","Mathematical","Problem","Solving","With","the","MATH","Dataset","''","by","Hendrycks","et","al",".","(","2021",")",":","https",":","\/\/arxiv.org\/pdf\/2103.03874.pdf","Example","input",",","using","official","examples",":","``","`","Given","a","mathematics","problem",",","determine","the","answer",".","Simplify","your","answer","as","much","as","possible",".","#","#","#","Problem",":","What","is","$","\\left","(","rac","{","7","}","{","8","}","ight",")","^3","\\cdot","\\left","(","rac","{","7","}","{","8","}","ight",")","^","{","-3","}","$","?","Answer",":","$","1","$","#","#","#","Problem",":","In","how","many","ways","can","4","books","be","selected","from","a","shelf","of","6","books","if","the","order","in","which","the","books","are","selected","does","not","matter","?","Answer",":","$","15","$","#","#","#","Problem",":","Find","the","distance","between","the","points","$","(","2,1",",","-4",")","$","and","$","(","5,8",",","-3",")",".","$","Answer",":","$","\\sqrt","{","59","}","$","#","#","#","Problem",":","The","faces","of","an","octahedral","die","are","labeled","with","digits","$","1","$","through","$","8","$",".","What","is","the","probability",",","expressed","as","a","common","fraction",",","of","rolling","a","sum","of","$","15","$","with","a","pair","of","such","octahedral","dice","?","Answer",":","$","rac","{","1","}","{","32","}","$","#","#","#","Problem",":","The","first","three","terms","of","an","arithmetic","sequence","are","1",",","10","and","19",",","respectively",".","What","is","the","value","of","the","21st","term","?","Answer",":","$","181","$","#","#","#","Problem",":","Calculate","$","6","\\cdot","8","rac","{","1","}","{","3","}","Answer",":","$","50","$","#","#","#","Problem",":","When","the","binary","number","$","100101110010_2","$","is","divided","by","4",",","what","is","the","remainder","(","give","your","answer","in","base","10",")","?","Answer",":","$","2","$","#","#","#","Problem",":","How","many","zeros","are","at","the","end","of","the","product","25","$","imes","$","240","?","Answer",":","$","3","$","#","#","#","Problem",":","What","is","$","\\dbinom","{","n","}","{","n","}","$","for","any","positive","integer","$","n","$","?","Answer",":","$","``","`","Example","expected","output","``","`","1","$","``","`"]
code: def get_instances(self) -> List[Instance]:
      dataset = {}
      data = typing.cast(
          DatasetDict,
          load_dataset("competition_math", ignore_verifications=True),
      )
  
      def group_by_key(dataset_list, key):
          dataset_per_key = collections.defaultdict(list)
          for ex in dataset_list:
              dataset_per_key[ex[key]].append(ex)
          return dataset_per_key
  
      instances = []
      for split, split_name in zip(
          [TRAIN_SPLIT, TEST_SPLIT], ["train", "test"]
      ):
          if split == TRAIN_SPLIT and self.use_official_examples:
              train_instances = [
                  (
                      "What is $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$?",
                      "1",
                  ),
                  (
                      "In how many ways can 4 books be selected from a shelf of 6 books"
                      + " if the order in which the books are selected does not matter?",
                      "15",
                  ),
                  (
                      "Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$",
                      "\sqrt{59}",
                  ),
                  (
                      "The faces of an octahedral die are labeled with digits $1$ through $8$."
                      + " What is the probability, expressed as a common fraction,"
                      + " of rolling a sum of $15$ with a pair of such octahedral dice?",
                      "\\frac{1}{32}",
                  ),
                  (
                      "The first three terms of an arithmetic sequence are 1, 10 and 19, respectively."
                      + " What is the value of the 21st term?",
                      "181",
                  ),
                  ("Calculate $6 \\cdot 8\\frac{1}{3}", "50"),
                  (
                      "When the binary number $100101110010_2$ is divided by 4,"
                      + " what is the remainder (give your answer in base 10)?",
                      "2",
                  ),
                  (
                      "How many zeros are at the end of the product 25 $\\times$ 240?",
                      "3",
                  ),
              ]
              dataset[TRAIN_SPLIT] = [
                  {"problem": problem, "answer": answer}
                  for problem, answer in train_instances
              ]
  
          else:
              examples: List[Dict[str, str]] = [
                  example for example in data[split_name]
              ]  # Filter by split
              examples = group_by_key(examples, "type")[
                  self.subject
              ]  # Filter by type or subject
              examples = group_by_key(examples, "level")[
                  self.level
              ]  # Filter by level
              dataset[split] = examples
  
              for example in dataset[split]:
                  # Sanity check that we filtered correctly
                  assert (
                      example["type"] == self.subject
                      and example["level"] == self.level
                  ), f"Wrong example was included after filtering: {example}"
  
                  if self.use_chain_of_thought:
                      answer = example["solution"]
                  else:
                      maybe_answer = get_answer(example["solution"])
                      if maybe_answer is None:
                          continue
                      answer = maybe_answer
                  example["answer"] = answer
  
          for example in dataset[split]:
              instance = Instance(
                  input=Input(text=example["problem"]),
                  references=[
                      Reference(
                          Output(text=example["answer"]),
                          tags=[CORRECT_TAG],
                      )
                  ],
                  split=split,
              )
              instances.append(instance)
  
      return instances
  
file_path: H2O/h2o_hf/helm/src/helm/benchmark/scenarios/math_scenario.py
imports: import collections
  import typing
  from typing import Dict, List, Optional
  from datasets import load_dataset, DatasetDict
  from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
label_desc: Use image node_id 2 for calling the MATHScenario obj's underlying member method code with example usage: obj.get_instances() and returns: instances, dataset_per_key