Upload eval_results/deepseek-ai/deepseek-math-7b-rl/main/math_deepseek_rl_cot/results_2024-07-01T14-36-07.008953.json with huggingface_hub

eval_results/deepseek-ai/deepseek-math-7b-rl/main/math_deepseek_rl_cot/results_2024-07-01T14-36-07.008953.json

@@ -0,0 +1,403 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 257347.55263128,
+    "end_time": 281700.066696367,
+    "total_evaluation_time_secondes": "24352.514065086987",
+    "model_name": "deepseek-ai/deepseek-math-7b-rl",
+    "model_sha": "f3cd419a172ff5a100e9a563ebfe7900d78e9740",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "12.87 GB",
+    "config": null
+  },
+  "results": {
+    "custom|math_deepseek_rl_cot:algebra|0": {
+      "qem": 0.6301600673967986,
+      "qem_stderr": 0.01401812717243806
+    },
+    "custom|math_deepseek_rl_cot:counting_and_probability|0": {
+      "qem": 0.34177215189873417,
+      "qem_stderr": 0.021808504852491584
+    },
+    "custom|math_deepseek_rl_cot:geometry|0": {
+      "qem": 0.3569937369519833,
+      "qem_stderr": 0.021914115773729835
+    },
+    "custom|math_deepseek_rl_cot:intermediate_algebra|0": {
+      "qem": 0.19601328903654486,
+      "qem_stderr": 0.013217944513396376
+    },
+    "custom|math_deepseek_rl_cot:number_theory|0": {
+      "qem": 0.31296296296296294,
+      "qem_stderr": 0.019972947695805383
+    },
+    "custom|math_deepseek_rl_cot:prealgebra|0": {
+      "qem": 0.6383467278989667,
+      "qem_stderr": 0.01628976770999433
+    },
+    "custom|math_deepseek_rl_cot:precalculus|0": {
+      "qem": 0.22527472527472528,
+      "qem_stderr": 0.017895005885612287
+    },
+    "custom|math_deepseek_rl_cot:_average|0": {
+      "qem": 0.3859319516315309,
+      "qem_stderr": 0.01787377337192398
+    },
+    "all": {
+      "qem": 0.3859319516315309,
+      "qem_stderr": 0.01787377337192398
+    }
+  },
+  "versions": {
+    "custom|math_deepseek_rl_cot:algebra|0": 0,
+    "custom|math_deepseek_rl_cot:counting_and_probability|0": 0,
+    "custom|math_deepseek_rl_cot:geometry|0": 0,
+    "custom|math_deepseek_rl_cot:intermediate_algebra|0": 0,
+    "custom|math_deepseek_rl_cot:number_theory|0": 0,
+    "custom|math_deepseek_rl_cot:prealgebra|0": 0,
+    "custom|math_deepseek_rl_cot:precalculus|0": 0
+  },
+  "config_tasks": {
+    "custom|math_deepseek_rl_cot:algebra": {
+      "name": "math_deepseek_rl_cot:algebra",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 1187,
+      "effective_num_docs": 1187,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_rl_cot:counting_and_probability": {
+      "name": "math_deepseek_rl_cot:counting_and_probability",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "counting_and_probability",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 474,
+      "effective_num_docs": 474,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_rl_cot:geometry": {
+      "name": "math_deepseek_rl_cot:geometry",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "geometry",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 479,
+      "effective_num_docs": 479,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_rl_cot:intermediate_algebra": {
+      "name": "math_deepseek_rl_cot:intermediate_algebra",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "intermediate_algebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 903,
+      "effective_num_docs": 903,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_rl_cot:number_theory": {
+      "name": "math_deepseek_rl_cot:number_theory",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "number_theory",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 540,
+      "effective_num_docs": 540,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_rl_cot:prealgebra": {
+      "name": "math_deepseek_rl_cot:prealgebra",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "prealgebra",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 871,
+      "effective_num_docs": 871,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    },
+    "custom|math_deepseek_rl_cot:precalculus": {
+      "name": "math_deepseek_rl_cot:precalculus",
+      "prompt_function": "math_deepseek_prompt_fn",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "precalculus",
+      "metric": [
+        "quasi_exact_match_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": null,
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "custom"
+      ],
+      "original_num_docs": 546,
+      "effective_num_docs": 546,
+      "trust_dataset": null,
+      "must_remove_duplicate_docs": null,
+      "version": 0
+    }
+  },
+  "summary_tasks": {
+    "custom|math_deepseek_rl_cot:algebra|0": {
+      "hashes": {
+        "hash_examples": "dcb01820b9d86e0d",
+        "hash_full_prompts": "7f1260779d95bf58",
+        "hash_input_tokens": "57158eeae9b6bf38",
+        "hash_cont_tokens": "09cfcbc3b65ecf48"
+      },
+      "truncated": 1187,
+      "non_truncated": 0,
+      "padded": 153,
+      "non_padded": 1034,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_rl_cot:counting_and_probability|0": {
+      "hashes": {
+        "hash_examples": "d72845695bf62192",
+        "hash_full_prompts": "1f67dad9a500dbb8",
+        "hash_input_tokens": "588febc753bee08c",
+        "hash_cont_tokens": "895a162596fa2068"
+      },
+      "truncated": 474,
+      "non_truncated": 0,
+      "padded": 51,
+      "non_padded": 423,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_rl_cot:geometry|0": {
+      "hashes": {
+        "hash_examples": "ec3e32d61c9774ce",
+        "hash_full_prompts": "00c7e19804742ce4",
+        "hash_input_tokens": "588d58229de15812",
+        "hash_cont_tokens": "be559a6a20337605"
+      },
+      "truncated": 479,
+      "non_truncated": 0,
+      "padded": 114,
+      "non_padded": 365,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_rl_cot:intermediate_algebra|0": {
+      "hashes": {
+        "hash_examples": "e22dffce2050ffaa",
+        "hash_full_prompts": "459d909fa8d3429a",
+        "hash_input_tokens": "3a1a54fee72e0a10",
+        "hash_cont_tokens": "701415421948964c"
+      },
+      "truncated": 903,
+      "non_truncated": 0,
+      "padded": 43,
+      "non_padded": 860,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_rl_cot:number_theory|0": {
+      "hashes": {
+        "hash_examples": "c87bdc11a396a831",
+        "hash_full_prompts": "6ca30019751b160b",
+        "hash_input_tokens": "3b361cd87ffc912c",
+        "hash_cont_tokens": "4e0c8752abc4928f"
+      },
+      "truncated": 537,
+      "non_truncated": 3,
+      "padded": 6,
+      "non_padded": 534,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_rl_cot:prealgebra|0": {
+      "hashes": {
+        "hash_examples": "28267717005d8d7d",
+        "hash_full_prompts": "f5100ec1648003c2",
+        "hash_input_tokens": "0a18706449a101d1",
+        "hash_cont_tokens": "33c6035b479b6819"
+      },
+      "truncated": 871,
+      "non_truncated": 0,
+      "padded": 53,
+      "non_padded": 818,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "custom|math_deepseek_rl_cot:precalculus|0": {
+      "hashes": {
+        "hash_examples": "40e85e93f5bf21d7",
+        "hash_full_prompts": "7c21366b20b4b9ea",
+        "hash_input_tokens": "c6f547e0dfd9f529",
+        "hash_cont_tokens": "44b1230b7034bf1b"
+      },
+      "truncated": 546,
+      "non_truncated": 0,
+      "padded": 21,
+      "non_padded": 525,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "825b967c0800ccea",
+      "hash_full_prompts": "af07312a5dbdfcb9",
+      "hash_input_tokens": "152bc219b338fc22",
+      "hash_cont_tokens": "8ce59928066af846"
+    },
+    "truncated": 4997,
+    "non_truncated": 3,
+    "padded": 441,
+    "non_padded": 4559,
+    "num_truncated_few_shots": 0
+  }
+}