Upload eval_results/deepseek-ai/deepseek-math-7b-rl/main/math_deepseek_cot/results_2024-07-01T02-38-10.632353.json with huggingface_hub

eval_results/deepseek-ai/deepseek-math-7b-rl/main/math_deepseek_cot/results_2024-07-01T02-38-10.632353.json

@@ -0,0 +1,403 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 253721.302661655,
+    "end_time": 278458.960923328,
+    "total_evaluation_time_secondes": "24737.658261673",
+    "model_name": "deepseek-ai/deepseek-math-7b-rl",
+    "model_sha": "f3cd419a172ff5a100e9a563ebfe7900d78e9740",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "12.87 GB",
+    "config": null
+  },
+  "results": {
+    "custom|math_deepseek_cot:algebra|0": {
+      "qem": 0.03201347935973041,
+      "qem_stderr": 0.00511162221827606
+    },
+    "custom|math_deepseek_cot:counting_and_probability|0": {
+      "qem": 0.004219409282700422,
+      "qem_stderr": 0.002980417365102051
+    },
+    "custom|math_deepseek_cot:geometry|0": {
+      "qem": 0.010438413361169102,
+      "qem_stderr": 0.004648627117184639
+    },
+    "custom|math_deepseek_cot:intermediate_algebra|0": {
+      "qem": 0.013289036544850499,
+      "qem_stderr": 0.003812751108019936
+    },
+    "custom|math_deepseek_cot:number_theory|0": {
+      "qem": 0.0,
+      "qem_stderr": 0.0
+    },
+    "custom|math_deepseek_cot:prealgebra|0": {
+      "qem": 0.04477611940298507,
+      "qem_stderr": 0.007011584710623336
+    },
+    "custom|math_deepseek_cot:precalculus|0": {
+      "qem": 0.020146520146520148,
+      "qem_stderr": 0.006018417889653956
+    },
+    "custom|math_deepseek_cot:_average|0": {
+      "qem": 0.017840425442565092,
+      "qem_stderr": 0.0042262029155514256
+    },
+    "all": {
+      "qem": 0.017840425442565092,
+      "qem_stderr": 0.0042262029155514256
+    }
+  },
+  "versions": {
+    "custom|math_deepseek_cot:algebra|0": 0,
+    "custom|math_deepseek_cot:counting_and_probability|0": 0,
+    "custom|math_deepseek_cot:geometry|0": 0,
+    "custom|math_deepseek_cot:intermediate_algebra|0": 0,
+    "custom|math_deepseek_cot:number_theory|0": 0,
+    "custom|math_deepseek_cot:prealgebra|0": 0,
+    "custom|math_deepseek_cot:precalculus|0": 0
+  },
+  "config_tasks": {
+    "custom|math_deepseek_cot:algebra": {
+      "name": "math_deepseek_cot:algebra",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 1187,
+      "effective_num_docs": 1187,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_cot:counting_and_probability": {
+      "name": "math_deepseek_cot:counting_and_probability",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "counting_and_probability",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 474,
+      "effective_num_docs": 474,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_cot:geometry": {
+      "name": "math_deepseek_cot:geometry",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "geometry",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 479,
+      "effective_num_docs": 479,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_cot:intermediate_algebra": {
+      "name": "math_deepseek_cot:intermediate_algebra",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "intermediate_algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 903,
+      "effective_num_docs": 903,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_cot:number_theory": {
+      "name": "math_deepseek_cot:number_theory",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "number_theory",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 540,
+      "effective_num_docs": 540,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_cot:prealgebra": {
+      "name": "math_deepseek_cot:prealgebra",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "prealgebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 871,
+      "effective_num_docs": 871,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_cot:precalculus": {
+      "name": "math_deepseek_cot:precalculus",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "precalculus",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 546,
+      "effective_num_docs": 546,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    }
+  },
+  "summary_tasks": {
+    "custom|math_deepseek_cot:algebra|0": {
+      "hashes": {
+        "hash_examples": "dd0c20b1f0cb4647",
+        "hash_full_prompts": "9dc2c8ae59782b3b",
+        "hash_input_tokens": "6c82f6e0f700d0dd",
+        "hash_cont_tokens": "8f7129ae00c1e7f4"
+      },
+      "truncated": 1187,
+      "non_truncated": 0,
+      "padded": 153,
+      "non_padded": 1034,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_cot:counting_and_probability|0": {
+      "hashes": {
+        "hash_examples": "155d950d14c44af4",
+        "hash_full_prompts": "0229e46c276fb642",
+        "hash_input_tokens": "8d6a6e01cba0c2df",
+        "hash_cont_tokens": "9e9011bd181065a6"
+      },
+      "truncated": 474,
+      "non_truncated": 0,
+      "padded": 51,
+      "non_padded": 423,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_cot:geometry|0": {
+      "hashes": {
+        "hash_examples": "d0903037eaaf129f",
+        "hash_full_prompts": "7ec2382c7aa27657",
+        "hash_input_tokens": "6be40ba81a92ef74",
+        "hash_cont_tokens": "557b92d91c772021"
+      },
+      "truncated": 479,
+      "non_truncated": 0,
+      "padded": 114,
+      "non_padded": 365,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_cot:intermediate_algebra|0": {
+      "hashes": {
+        "hash_examples": "11e643d9ae66ed0e",
+        "hash_full_prompts": "35b30af62390119f",
+        "hash_input_tokens": "bc5244afa75043dc",
+        "hash_cont_tokens": "f95c183c254a69cb"
+      },
+      "truncated": 903,
+      "non_truncated": 0,
+      "padded": 43,
+      "non_padded": 860,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_cot:number_theory|0": {
+      "hashes": {
+        "hash_examples": "d5cb46b7f8cc037c",
+        "hash_full_prompts": "d35d76ee7b76069c",
+        "hash_input_tokens": "2898f6f86f93fd51",
+        "hash_cont_tokens": "391f851a0533e193"
+      },
+      "truncated": 537,
+      "non_truncated": 3,
+      "padded": 6,
+      "non_padded": 534,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_cot:prealgebra|0": {
+      "hashes": {
+        "hash_examples": "927b75048eaa22ce",
+        "hash_full_prompts": "b44563e8a610a5e0",
+        "hash_input_tokens": "d61848981f04e253",
+        "hash_cont_tokens": "22cca59bb15fdf81"
+      },
+      "truncated": 871,
+      "non_truncated": 0,
+      "padded": 53,
+      "non_padded": 818,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_cot:precalculus|0": {
+      "hashes": {
+        "hash_examples": "a2ad19d3f21c8c63",
+        "hash_full_prompts": "16ec0e775cc4e656",
+        "hash_input_tokens": "6545a1bd9ce71e45",
+        "hash_cont_tokens": "ee65a5af8a844194"
+      },
+      "truncated": 546,
+      "non_truncated": 0,
+      "padded": 21,
+      "non_padded": 525,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "0aa3308517bd659c",
+      "hash_full_prompts": "cb8f5223e4e35bcc",
+      "hash_input_tokens": "24e1699323938802",
+      "hash_cont_tokens": "e71cd19e1d2d55b1"
+    },
+    "truncated": 4997,
+    "non_truncated": 3,
+    "padded": 441,
+    "non_padded": 4559,
+    "num_truncated_few_shots": 0
+  }
+}