Upload eval_results/AI-MO/mistral-7b-sft/aimo_v03.00/mini_math/results_2024-04-23T12-07-13.400858.json with huggingface_hub

eval_results/AI-MO/mistral-7b-sft/aimo_v03.00/mini_math/results_2024-04-23T12-07-13.400858.json

@@ -0,0 +1,291 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 1691110.31978568,
+    "end_time": 1692074.373145221,
+    "total_evaluation_time_secondes": "964.0533595411107",
+    "model_name": "AI-MO/mistral-7b-sft",
+    "model_sha": "159047b1ab76bbb7c9369ee71bfef1d441fc029e",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "13.99 GB",
+    "config": null
+  },
+  "results": {
+    "custom|mini_math:level_1|0": {
+      "qem": 0.6842105263157895,
+      "qem_stderr": 0.07641750223933595
+    },
+    "custom|mini_math:level_2|0": {
+      "qem": 0.4225352112676056,
+      "qem_stderr": 0.05903984205682581
+    },
+    "custom|mini_math:level_3|0": {
+      "qem": 0.28205128205128205,
+      "qem_stderr": 0.051282051282051246
+    },
+    "custom|mini_math:level_4|0": {
+      "qem": 0.24770642201834864,
+      "qem_stderr": 0.04153846393641014
+    },
+    "custom|mini_math:level_5|0": {
+      "qem": 0.06542056074766354,
+      "qem_stderr": 0.02401664339953487
+    },
+    "custom|mini_math:_average|0": {
+      "qem": 0.34038480048013786,
+      "qem_stderr": 0.0504589005828316
+    },
+    "all": {
+      "qem": 0.34038480048013786,
+      "qem_stderr": 0.0504589005828316
+    }
+  },
+  "versions": {
+    "custom|mini_math:level_1|0": 0,
+    "custom|mini_math:level_2|0": 0,
+    "custom|mini_math:level_3|0": 0,
+    "custom|mini_math:level_4|0": 0,
+    "custom|mini_math:level_5|0": 0
+  },
+  "config_tasks": {
+    "custom|mini_math:level_1": {
+      "name": "mini_math:level_1",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 1",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 38,
+      "effective_num_docs": 38,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_2": {
+      "name": "mini_math:level_2",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 2",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 71,
+      "effective_num_docs": 71,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_3": {
+      "name": "mini_math:level_3",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 3",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 78,
+      "effective_num_docs": 78,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_4": {
+      "name": "mini_math:level_4",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 4",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 109,
+      "effective_num_docs": 109,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    },
+    "custom|mini_math:level_5": {
+      "name": "mini_math:level_5",
+      "prompt_function": "math",
+      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
+      "hf_subset": "Level 5",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "frozen": false,
+      "suite": [
+        "custom",
+        "mini_math"
+      ],
+      "original_num_docs": 107,
+      "effective_num_docs": 107,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null
+    }
+  },
+  "summary_tasks": {
+    "custom|mini_math:level_1|0": {
+      "hashes": {
+        "hash_examples": "781a25d9a2f29e87",
+        "hash_full_prompts": "ab476c3f1e6189c8",
+        "hash_input_tokens": "748b0dfb1d890562",
+        "hash_cont_tokens": "719d7c50adc91f8f"
+      },
+      "truncated": 38,
+      "non_truncated": 0,
+      "padded": 23,
+      "non_padded": 15,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_2|0": {
+      "hashes": {
+        "hash_examples": "471a4a8568c70994",
+        "hash_full_prompts": "8ec479c7a8a3d59b",
+        "hash_input_tokens": "79bfecc8111dea84",
+        "hash_cont_tokens": "238f82a182eb52a4"
+      },
+      "truncated": 71,
+      "non_truncated": 0,
+      "padded": 35,
+      "non_padded": 36,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_3|0": {
+      "hashes": {
+        "hash_examples": "aa6979a90c9a7776",
+        "hash_full_prompts": "0c55bb3e608c2221",
+        "hash_input_tokens": "152332152ec83113",
+        "hash_cont_tokens": "15e129814ba3f8ce"
+      },
+      "truncated": 78,
+      "non_truncated": 0,
+      "padded": 36,
+      "non_padded": 42,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_4|0": {
+      "hashes": {
+        "hash_examples": "2e9e3be067de6ec6",
+        "hash_full_prompts": "e471b8649404f9a7",
+        "hash_input_tokens": "b0305a77648ed6a6",
+        "hash_cont_tokens": "0d133d0776517b35"
+      },
+      "truncated": 108,
+      "non_truncated": 1,
+      "padded": 36,
+      "non_padded": 73,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|mini_math:level_5|0": {
+      "hashes": {
+        "hash_examples": "18b402b510f9a746",
+        "hash_full_prompts": "12e58a2a352d2763",
+        "hash_input_tokens": "5c4b6f568f76445a",
+        "hash_cont_tokens": "c138c90600d234a1"
+      },
+      "truncated": 107,
+      "non_truncated": 0,
+      "padded": 39,
+      "non_padded": 68,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "470ce01cab0383c9",
+      "hash_full_prompts": "3a272d494f912b3f",
+      "hash_input_tokens": "7d3b36c2ba40899d",
+      "hash_cont_tokens": "017293ac30be3dc4"
+    },
+    "truncated": 402,
+    "non_truncated": 1,
+    "padded": 169,
+    "non_padded": 234,
+    "num_truncated_few_shots": 0
+  }
+}