Upload eval_results/AI-MO/mixtral-47b-sft/aimo_v00.01/math_v2/results_2024-04-29T14-03-18.023791.json with huggingface_hub

eval_results/AI-MO/mixtral-47b-sft/aimo_v00.01/math_v2/results_2024-04-29T14-03-18.023791.json

@@ -0,0 +1,389 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 190375.359157276,
+    "end_time": 276615.230493667,
+    "total_evaluation_time_secondes": "86239.87133639102",
+    "model_name": "AI-MO/mixtral-47b-sft",
+    "model_sha": "9f3ead81e0a9c2d74a04a83d24d9d8f6cb6d4826",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "87.49 GB",
+    "config": null
+  },
+  "results": {
+    "custom|math_v2:algebra|0": {
+      "qem": 0.4667228306655434,
+      "qem_stderr": 0.014486506776458814
+    },
+    "custom|math_v2:counting_and_probability|0": {
+      "qem": 0.27848101265822783,
+      "qem_stderr": 0.020610622389419315
+    },
+    "custom|math_v2:geometry|0": {
+      "qem": 0.2567849686847599,
+      "qem_stderr": 0.01998150025896483
+    },
+    "custom|math_v2:intermediate_algebra|0": {
+      "qem": 0.14174972314507198,
+      "qem_stderr": 0.011613545265293153
+    },
+    "custom|math_v2:number_theory|0": {
+      "qem": 0.20925925925925926,
+      "qem_stderr": 0.01752124719563077
+    },
+    "custom|math_v2:prealgebra|0": {
+      "qem": 0.5361653272101033,
+      "qem_stderr": 0.016907186430552387
+    },
+    "custom|math_v2:precalculus|0": {
+      "qem": 0.1794871794871795,
+      "qem_stderr": 0.016438460824782845
+    },
+    "custom|math_v2:_average|0": {
+      "qem": 0.2955214715871636,
+      "qem_stderr": 0.01679415273444316
+    },
+    "all": {
+      "qem": 0.2955214715871636,
+      "qem_stderr": 0.01679415273444316
+    }
+  },
+  "versions": {
+    "custom|math_v2:algebra|0": 0,
+    "custom|math_v2:counting_and_probability|0": 0,
+    "custom|math_v2:geometry|0": 0,
+    "custom|math_v2:intermediate_algebra|0": 0,
+    "custom|math_v2:number_theory|0": 0,
+    "custom|math_v2:prealgebra|0": 0,
+    "custom|math_v2:precalculus|0": 0
+  },
+  "config_tasks": {
+    "custom|math_v2:algebra": {
+      "name": "math_v2:algebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 1187,
+      "effective_num_docs": 1187,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:counting_and_probability": {
+      "name": "math_v2:counting_and_probability",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "counting_and_probability",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 474,
+      "effective_num_docs": 474,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:geometry": {
+      "name": "math_v2:geometry",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "geometry",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 479,
+      "effective_num_docs": 479,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:intermediate_algebra": {
+      "name": "math_v2:intermediate_algebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "intermediate_algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 903,
+      "effective_num_docs": 903,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:number_theory": {
+      "name": "math_v2:number_theory",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "number_theory",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 540,
+      "effective_num_docs": 540,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:prealgebra": {
+      "name": "math_v2:prealgebra",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "prealgebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 871,
+      "effective_num_docs": 871,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|math_v2:precalculus": {
+      "name": "math_v2:precalculus",
+      "prompt_function": "math_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "precalculus",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 546,
+      "effective_num_docs": 546,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    }
+  },
+  "summary_tasks": {
+    "custom|math_v2:algebra|0": {
+      "hashes": {
+        "hash_examples": "6ec951c5aa417d2a",
+        "hash_full_prompts": "4e0b4b752e408da7",
+        "hash_input_tokens": "5c933039b7f4c907",
+        "hash_cont_tokens": "2308663b30b9a53d"
+      },
+      "truncated": 1187,
+      "non_truncated": 0,
+      "padded": 186,
+      "non_padded": 1001,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:counting_and_probability|0": {
+      "hashes": {
+        "hash_examples": "cd34cb03dc09e1ad",
+        "hash_full_prompts": "9ebbc169a1089204",
+        "hash_input_tokens": "ff493ce6cceb8548",
+        "hash_cont_tokens": "3adf38220b16b399"
+      },
+      "truncated": 474,
+      "non_truncated": 0,
+      "padded": 42,
+      "non_padded": 432,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:geometry|0": {
+      "hashes": {
+        "hash_examples": "e1011f83d0cb54d0",
+        "hash_full_prompts": "e6e4d9f63cdecf28",
+        "hash_input_tokens": "f61c2560fd69fc44",
+        "hash_cont_tokens": "8baa9f6ebd25b83a"
+      },
+      "truncated": 479,
+      "non_truncated": 0,
+      "padded": 129,
+      "non_padded": 350,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:intermediate_algebra|0": {
+      "hashes": {
+        "hash_examples": "aa72155be072b11c",
+        "hash_full_prompts": "38dad98a61e4f0ed",
+        "hash_input_tokens": "024718b19ffe7955",
+        "hash_cont_tokens": "d0db1bccc9ecb8fb"
+      },
+      "truncated": 900,
+      "non_truncated": 3,
+      "padded": 54,
+      "non_padded": 849,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:number_theory|0": {
+      "hashes": {
+        "hash_examples": "b8565befcdbe9247",
+        "hash_full_prompts": "8ca7a4c78d6e1280",
+        "hash_input_tokens": "e7eb3cb8d452018e",
+        "hash_cont_tokens": "4b05ec61fa4c587b"
+      },
+      "truncated": 540,
+      "non_truncated": 0,
+      "padded": 7,
+      "non_padded": 533,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:prealgebra|0": {
+      "hashes": {
+        "hash_examples": "e04d1527fe369f16",
+        "hash_full_prompts": "51e0c4e3d62158e4",
+        "hash_input_tokens": "abf3673501aec1ab",
+        "hash_cont_tokens": "3a5c994ef68adac3"
+      },
+      "truncated": 871,
+      "non_truncated": 0,
+      "padded": 55,
+      "non_padded": 816,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_v2:precalculus|0": {
+      "hashes": {
+        "hash_examples": "97606c134f223253",
+        "hash_full_prompts": "2438e95fc348976e",
+        "hash_input_tokens": "33a48df19e37782a",
+        "hash_cont_tokens": "256da5159c75776c"
+      },
+      "truncated": 546,
+      "non_truncated": 0,
+      "padded": 24,
+      "non_padded": 522,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "de5f0d623d1896c2",
+      "hash_full_prompts": "3cb45e93275657cb",
+      "hash_input_tokens": "6a387a67e9deefce",
+      "hash_cont_tokens": "99867db5ad909ba9"
+    },
+    "truncated": 4997,
+    "non_truncated": 3,
+    "padded": 497,
+    "non_padded": 4503,
+    "num_truncated_few_shots": 0
+  }
+}