Upload eval_results/AI-MO/mistral-7b-sft/aimo_v03.01/math_v2/results_2024-04-27T07-57-00.481918.json with huggingface_hub

eval_results/AI-MO/mistral-7b-sft/aimo_v03.01/math_v2/results_2024-04-27T07-57-00.481918.json

@@ -0,0 +1,389 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 1404883.661664768,
+    "end_time": 1414730.017814066,
+    "total_evaluation_time_secondes": "9846.356149298139",
+    "model_name": "AI-MO/mistral-7b-sft",
+    "model_sha": "147a5797ae74bc37a561429745b8781fde7e9eff",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "13.99 GB",
+    "config": null
+  },
+  "results": {
+    "custom|math_v2:algebra|0": {
+      "qem": 0.37068239258635216,
+      "qem_stderr": 0.014024699857095878
+    },
+    "custom|math_v2:counting_and_probability|0": {
+      "qem": 0.12236286919831224,
+      "qem_stderr": 0.015067866025208529
+    },
+    "custom|math_v2:geometry|0": {
+      "qem": 0.1336116910229645,
+      "qem_stderr": 0.015561969995340364
+    },
+    "custom|math_v2:intermediate_algebra|0": {
+      "qem": 0.10188261351052048,
+      "qem_stderr": 0.010071944446768333
+    },
+    "custom|math_v2:number_theory|0": {
+      "qem": 0.08148148148148149,
+      "qem_stderr": 0.011783628281121668
+    },
+    "custom|math_v2:prealgebra|0": {
+      "qem": 0.3157290470723307,
+      "qem_stderr": 0.015758384592042537
+    },
+    "custom|math_v2:precalculus|0": {
+      "qem": 0.11172161172161173,
+      "qem_stderr": 0.013494130099732627
+    },
+    "custom|math_v2:_average|0": {
+      "qem": 0.17678167237051048,
+      "qem_stderr": 0.013680374756758564
+    },
+    "all": {
+      "qem": 0.17678167237051048,
+      "qem_stderr": 0.013680374756758564
+    }
+  },
+  "versions": {
+    "custom|math_v2:algebra|0": 0,
+    "custom|math_v2:counting_and_probability|0": 0,
+    "custom|math_v2:geometry|0": 0,
+    "custom|math_v2:intermediate_algebra|0": 0,
+    "custom|math_v2:number_theory|0": 0,
+    "custom|math_v2:prealgebra|0": 0,
+    "custom|math_v2:precalculus|0": 0
+  },
+  "config_tasks": {
+    "custom|math_v2:algebra": {
+      "name": "math_v2:algebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 1187,
+      "effective_num_docs": 1187,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:counting_and_probability": {
+      "name": "math_v2:counting_and_probability",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "counting_and_probability",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 474,
+      "effective_num_docs": 474,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:geometry": {
+      "name": "math_v2:geometry",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "geometry",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 479,
+      "effective_num_docs": 479,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:intermediate_algebra": {
+      "name": "math_v2:intermediate_algebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "intermediate_algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 903,
+      "effective_num_docs": 903,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:number_theory": {
+      "name": "math_v2:number_theory",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "number_theory",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 540,
+      "effective_num_docs": 540,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:prealgebra": {
+      "name": "math_v2:prealgebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "prealgebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 871,
+      "effective_num_docs": 871,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:precalculus": {
+      "name": "math_v2:precalculus",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "precalculus",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 546,
+      "effective_num_docs": 546,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    }
+  },
+  "summary_tasks": {
+    "custom|math_v2:algebra|0": {
+      "hashes": {
+        "hash_examples": "6ec951c5aa417d2a",
+        "hash_full_prompts": "4e0b4b752e408da7",
+        "hash_input_tokens": "40949aee162d5a25",
+        "hash_cont_tokens": "29a9ade9e1d83253"
+      },
+      "truncated": 1187,
+      "non_truncated": 0,
+      "padded": 186,
+      "non_padded": 1001,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:counting_and_probability|0": {
+      "hashes": {
+        "hash_examples": "cd34cb03dc09e1ad",
+        "hash_full_prompts": "9ebbc169a1089204",
+        "hash_input_tokens": "671cb933a0ed9e9f",
+        "hash_cont_tokens": "e5f65cf153a406ac"
+      },
+      "truncated": 474,
+      "non_truncated": 0,
+      "padded": 42,
+      "non_padded": 432,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:geometry|0": {
+      "hashes": {
+        "hash_examples": "e1011f83d0cb54d0",
+        "hash_full_prompts": "e6e4d9f63cdecf28",
+        "hash_input_tokens": "6ebe67f495eb0f04",
+        "hash_cont_tokens": "e0607eaa2ee9e7e0"
+      },
+      "truncated": 479,
+      "non_truncated": 0,
+      "padded": 129,
+      "non_padded": 350,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:intermediate_algebra|0": {
+      "hashes": {
+        "hash_examples": "aa72155be072b11c",
+        "hash_full_prompts": "38dad98a61e4f0ed",
+        "hash_input_tokens": "6a2208475c1766ea",
+        "hash_cont_tokens": "82c174ea7ce7c1fe"
+      },
+      "truncated": 900,
+      "non_truncated": 3,
+      "padded": 54,
+      "non_padded": 849,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:number_theory|0": {
+      "hashes": {
+        "hash_examples": "b8565befcdbe9247",
+        "hash_full_prompts": "8ca7a4c78d6e1280",
+        "hash_input_tokens": "40d9f5eaae183f33",
+        "hash_cont_tokens": "426355cc2a0d49dc"
+      },
+      "truncated": 540,
+      "non_truncated": 0,
+      "padded": 7,
+      "non_padded": 533,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:prealgebra|0": {
+      "hashes": {
+        "hash_examples": "e04d1527fe369f16",
+        "hash_full_prompts": "51e0c4e3d62158e4",
+        "hash_input_tokens": "c5bd83691493e671",
+        "hash_cont_tokens": "44fdef4732242a33"
+      },
+      "truncated": 871,
+      "non_truncated": 0,
+      "padded": 55,
+      "non_padded": 816,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:precalculus|0": {
+      "hashes": {
+        "hash_examples": "97606c134f223253",
+        "hash_full_prompts": "2438e95fc348976e",
+        "hash_input_tokens": "b860ff9d92ae1d85",
+        "hash_cont_tokens": "0ce410cf76391189"
+      },
+      "truncated": 546,
+      "non_truncated": 0,
+      "padded": 24,
+      "non_padded": 522,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "de5f0d623d1896c2",
+      "hash_full_prompts": "3cb45e93275657cb",
+      "hash_input_tokens": "e3bb985d4c1b1522",
+      "hash_cont_tokens": "391a0a352c5eff15"
+    },
+    "truncated": 4997,
+    "non_truncated": 3,
+    "padded": 497,
+    "non_padded": 4503,
+    "num_truncated_few_shots": 0
+  }
+}