class_docstr: Synthetic Reasoning benchmark inspired by
  "LIME: Learning Inductive Bias for Primitives of Mathematical Reasoning"
      https://arxiv.org/abs/2101.06223
class_docstr_tok: ["Synthetic","Reasoning","benchmark","inspired","by","''","LIME",":","Learning","Inductive","Bias","for","Primitives","of","Mathematical","Reasoning","''","https",":","\/\/arxiv.org\/abs\/2101.06223"]
code: def get_instances(self) -> List[Instance]:
      # We fix the seed for data generation to ensure reproducibility.
      # Read all the instances
      instances: List[Instance] = []
  
      rule_symbols = RULE_SYMBOLS
      tokens = ANIMALS + FRUITS
      math_symbols = MATH_SYMBOLS
  
      for sample_idx in range(
          self.num_train_instances
          + self.num_val_instances
          + self.num_test_instances
      ):
          # Sample rule symbols
          sampled_rule_symbols = list(
              self.rng.choice(rule_symbols, size=self.rng.randint(2, 4))
          )
          sampled_rule_symbols_set = sorted(
              list(set(sampled_rule_symbols))
          )  # sorted to make it deterministic
  
          # Sample math symbols
          sampled_math_symbols = list(
              self.rng.choice(math_symbols, size=self.rng.randint(2, 4))
          )
  
          # generate the pattern
          pattern = self.gen_pattern(
              sampled_math_symbols, sampled_rule_symbols
          )
  
          # generate one substitution
          substitute_dict, substitute_dict_str = self.gen_subst(
              sampled_rule_symbols_set, tokens
          )
          result = pattern_subst(
              pattern, sampled_rule_symbols_set, substitute_dict
          )
  
          # generate another substitution
          substitute_dict_2, _ = self.gen_subst(
              sampled_rule_symbols_set, tokens
          )
          result_2 = pattern_subst(
              pattern, sampled_rule_symbols_set, substitute_dict_2
          )
  
          result_string = " ".join(result)
          pattern_string = " ".join(pattern)
  
          src: str
          tgt: str
          if self.mode == "induction":
              result_string_2 = " ".join(result_2)
              src = f"Two results: {result_string} | {result_string_2}"
              tgt = f"Rule: {pattern_string}"
          elif self.mode == "variable_substitution":
              src = f"Rules: {pattern_string} | Substitutions: {substitute_dict_str}"
              tgt = " ".join(result)
          elif self.mode == "pattern_match":
              # we sample 3 other pattern strings as negatives for patterns matching.
              other_patterns = [
                  " ".join(
                      self.gen_pattern(
                          sampled_math_symbols, sampled_rule_symbols_set
                      )
                  )
                  for _ in range(3)
              ]
              all_patterns = other_patterns + [pattern_string]
              self.rng.shuffle(all_patterns)
              all_pattern_string = " | ".join(all_patterns)
              src = f"Rules: {all_pattern_string} | Result: {result_string}"
              tgt = pattern_string
          else:
              raise ValueError(f"Invalid mode: {self.mode}")
  
          split: str
          if sample_idx < self.num_train_instances:
              split = TRAIN_SPLIT
          elif (
              sample_idx
              < self.num_train_instances + self.num_val_instances
          ):
              split = VALID_SPLIT
          else:
              split = TEST_SPLIT
  
          instance = Instance(
              input=Input(text=src),
              references=[
                  Reference(Output(text=tgt), tags=[CORRECT_TAG])
              ],
              split=split,
          )
          instances.append(instance)
  
      return instances
  
file_path: H2O/h2o_hf/helm/src/helm/benchmark/scenarios/synthetic_reasoning_scenario.py
imports: import numpy
  from typing import List, Dict, Tuple
  from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
label_desc: Use image node_id 4 for calling the SyntheticReasoningScenario obj's underlying member method code with example usage: obj.get_instances() and returns: instances