Clean up false PoT
Browse filesThis view is limited to 50 files because it contains too many changes.
See raw diff
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_hard_mmos/results_2024-05-24T09-30-06.964431.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_hard_pot/results_2024-05-27T14-37-04.090977.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_medium_mmos/results_2024-05-24T09-29-31.249987.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_medium_pot/results_2024-05-27T14-31-39.435993.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_mmos/results_2024-05-24T09-38-01.891596.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_pot/results_2024-05-27T12-10-10.985244.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_pot/results_2024-05-27T12-38-20.637098.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_pot/results_2024-05-27T13-19-13.846655.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_mmos/results_2024-05-24T09-36-20.897469.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_pot/results_2024-05-27T12-08-02.630027.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_pot/results_2024-05-27T12-36-15.235885.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_pot/results_2024-05-27T13-10-03.543903.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_hard_mmos/results_2024-05-24T09-36-06.066222.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_hard_pot/results_2024-05-27T14-35-43.671086.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_medium_mmos/results_2024-05-24T09-36-32.170044.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_medium_pot/results_2024-05-27T14-31-11.406681.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_hard_mmos/results_2024-05-24T09-48-10.979290.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_hard_pot/results_2024-05-27T14-35-53.533059.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_medium_mmos/results_2024-05-24T09-48-12.415661.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_medium_pot/results_2024-05-27T14-30-36.611690.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.13/aimo_kaggle_hard_mmos/results_2024-05-24T10-07-40.764237.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.13/aimo_kaggle_medium_mmos/results_2024-05-24T10-07-11.135359.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.14/aimo_kaggle_hard_mmos/results_2024-05-24T10-16-49.966984.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.14/aimo_kaggle_medium_mmos/results_2024-05-24T10-16-38.488727.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.15/aimo_kaggle_hard_mmos/results_2024-05-24T10-25-09.540279.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.15/aimo_kaggle_medium_mmos/results_2024-05-24T10-25-14.707607.json +0 -91
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.01/aimo_kaggle_hard_pot/results_2024-05-27T22-52-32.480755.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.01/aimo_kaggle_medium_pot/results_2024-05-27T22-49-32.235858.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.02/aimo_kaggle_hard_pot/results_2024-05-27T23-01-26.617019.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.02/aimo_kaggle_medium_pot/results_2024-05-27T22-56-40.113739.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.03/aimo_kaggle_hard_pot/results_2024-05-27T22-50-33.482970.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.03/aimo_kaggle_medium_pot/results_2024-05-27T22-46-20.885308.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.04/aimo_kaggle_hard_pot/results_2024-05-27T23-02-16.395548.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.04/aimo_kaggle_medium_pot/results_2024-05-27T22-56-39.512900.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.05/aimo_kaggle_hard_pot/results_2024-05-27T22-47-26.728707.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.05/aimo_kaggle_medium_pot/results_2024-05-27T22-42-48.691321.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.06/aimo_kaggle_hard_pot/results_2024-05-27T23-22-45.715568.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.06/aimo_kaggle_medium_pot/results_2024-05-27T23-16-30.855912.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.07/aimo_kaggle_hard_pot/results_2024-05-27T22-43-29.392522.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.07/aimo_kaggle_medium_pot/results_2024-05-27T22-39-34.119815.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.08/aimo_kaggle_hard_pot/results_2024-05-27T22-45-02.958180.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.08/aimo_kaggle_medium_pot/results_2024-05-27T22-39-55.232091.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.09/aimo_kaggle_hard_pot/results_2024-05-27T23-01-23.916419.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.09/aimo_kaggle_medium_pot/results_2024-05-27T22-56-49.561125.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.10/aimo_kaggle_hard_pot/results_2024-05-27T22-56-26.205133.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.10/aimo_kaggle_medium_pot/results_2024-05-27T22-52-23.993586.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.11/aimo_kaggle_hard_pot/results_2024-05-27T23-28-10.769733.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.11/aimo_kaggle_medium_pot/results_2024-05-27T23-24-01.746924.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.12/aimo_kaggle_hard_pot/results_2024-05-27T22-58-24.076261.json +0 -193
- eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.12/aimo_kaggle_medium_pot/results_2024-05-27T22-54-41.166514.json +0 -193
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_hard_mmos/results_2024-05-24T09-30-06.964431.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 857446.223097524,
|
| 9 |
-
"end_time": 857817.902641332,
|
| 10 |
-
"total_evaluation_time_secondes": "371.6795438079862",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "e9f1f0034b3f35397651713c744779b27c7b0d2f",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.027994168488950606
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.04,
|
| 24 |
-
"qem_stderr": 0.027994168488950606
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_hard_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "b40b6a493a95bf77",
|
| 66 |
-
"hash_full_prompts": "bf4ae1359925ab3b",
|
| 67 |
-
"hash_input_tokens": "5f11181c1839f9c3",
|
| 68 |
-
"hash_cont_tokens": "52b2b59de598fdd3"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "79dbebcff6acad9e",
|
| 81 |
-
"hash_full_prompts": "86a15c8410a494ef",
|
| 82 |
-
"hash_input_tokens": "28eff78f8cb1ac9f",
|
| 83 |
-
"hash_cont_tokens": "cc9bd188ea6976bc"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_hard_pot/results_2024-05-27T14-37-04.090977.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 1143715.699540001,
|
| 9 |
-
"end_time": 1144501.128154096,
|
| 10 |
-
"total_evaluation_time_secondes": "785.4286140948534",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "e9f1f0034b3f35397651713c744779b27c7b0d2f",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.06,
|
| 20 |
-
"qem_stderr": 0.0339266916772512
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.48,
|
| 24 |
-
"qem_stderr": 0.07137140569598172
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.56,
|
| 28 |
-
"qem_stderr": 0.07091242083423345
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.3666666666666667,
|
| 32 |
-
"qem_stderr": 0.058736839402488784
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.3666666666666667,
|
| 36 |
-
"qem_stderr": 0.058736839402488784
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "b73fa39a0f0c8082"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "1b2f354c5d65f92d"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "ddd3e2af9db6590b"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "a225423cb032047d"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_medium_mmos/results_2024-05-24T09-29-31.249987.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 727962.553320119,
|
| 9 |
-
"end_time": 728298.349595936,
|
| 10 |
-
"total_evaluation_time_secondes": "335.79627581697423",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "e9f1f0034b3f35397651713c744779b27c7b0d2f",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 19 |
-
"qem": 0.075,
|
| 20 |
-
"qem_stderr": 0.04217636961434869
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.075,
|
| 24 |
-
"qem_stderr": 0.04217636961434869
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_medium_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "3401efda8b0cbcb5",
|
| 66 |
-
"hash_full_prompts": "9553e2c8c7c87406",
|
| 67 |
-
"hash_input_tokens": "4f9c9b13fc9c2736",
|
| 68 |
-
"hash_cont_tokens": "60bf7b05082d037a"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "4c81d27cfdb9d737",
|
| 81 |
-
"hash_full_prompts": "466bf00fd3e3d325",
|
| 82 |
-
"hash_input_tokens": "f3be8baebf6e5c0d",
|
| 83 |
-
"hash_cont_tokens": "572a44bb3cfcb49a"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.09/aimo_kaggle_medium_pot/results_2024-05-27T14-31-39.435993.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 438052.255329886,
|
| 9 |
-
"end_time": 438511.635606925,
|
| 10 |
-
"total_evaluation_time_secondes": "459.3802770390175",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "e9f1f0034b3f35397651713c744779b27c7b0d2f",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.125,
|
| 20 |
-
"qem_stderr": 0.05295740910852021
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.15,
|
| 24 |
-
"qem_stderr": 0.05717718748968655
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.2,
|
| 28 |
-
"qem_stderr": 0.06405126152203487
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.15833333333333335,
|
| 32 |
-
"qem_stderr": 0.05806195270674721
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.15833333333333335,
|
| 36 |
-
"qem_stderr": 0.05806195270674721
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "d3d0c2f5914eea85"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "70097c50fd00a4cd"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "a052161a23e608a2"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "480ba89e1c3b08f4"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_mmos/results_2024-05-24T09-38-01.891596.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2115203.534425676,
|
| 9 |
-
"end_time": 2115616.753860881,
|
| 10 |
-
"total_evaluation_time_secondes": "413.21943520521745",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.027994168488950612
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.04,
|
| 24 |
-
"qem_stderr": 0.027994168488950612
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_hard_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "b40b6a493a95bf77",
|
| 66 |
-
"hash_full_prompts": "bf4ae1359925ab3b",
|
| 67 |
-
"hash_input_tokens": "5f11181c1839f9c3",
|
| 68 |
-
"hash_cont_tokens": "70ab15813457a0e7"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "79dbebcff6acad9e",
|
| 81 |
-
"hash_full_prompts": "86a15c8410a494ef",
|
| 82 |
-
"hash_input_tokens": "28eff78f8cb1ac9f",
|
| 83 |
-
"hash_cont_tokens": "932433668b9d10d1"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_pot/results_2024-05-27T12-10-10.985244.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 4049391.660600598,
|
| 9 |
-
"end_time": 4049852.921236317,
|
| 10 |
-
"total_evaluation_time_secondes": "461.26063571916893",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.08,
|
| 20 |
-
"qem_stderr": 0.038756171332144415
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.08,
|
| 24 |
-
"qem_stderr": 0.038756171332144415
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 33 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code_and_math"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "303213a38d9f7512",
|
| 66 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 67 |
-
"hash_input_tokens": "69dabd3aaebed332",
|
| 68 |
-
"hash_cont_tokens": "38540998e511c7fa"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "cae48890a7c47904",
|
| 81 |
-
"hash_full_prompts": "6f120f8d8bb938e6",
|
| 82 |
-
"hash_input_tokens": "ff7094ff98f5f808",
|
| 83 |
-
"hash_cont_tokens": "d081df183e2788f4"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_pot/results_2024-05-27T12-38-20.637098.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 9223358.910966022,
|
| 9 |
-
"end_time": 9223818.340137284,
|
| 10 |
-
"total_evaluation_time_secondes": "459.42917126230896",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.08,
|
| 20 |
-
"qem_stderr": 0.038756171332144415
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.08,
|
| 24 |
-
"qem_stderr": 0.038756171332144415
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 33 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code_and_math"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "303213a38d9f7512",
|
| 66 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 67 |
-
"hash_input_tokens": "69dabd3aaebed332",
|
| 68 |
-
"hash_cont_tokens": "38540998e511c7fa"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "cae48890a7c47904",
|
| 81 |
-
"hash_full_prompts": "6f120f8d8bb938e6",
|
| 82 |
-
"hash_input_tokens": "ff7094ff98f5f808",
|
| 83 |
-
"hash_cont_tokens": "d081df183e2788f4"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_hard_pot/results_2024-05-27T13-19-13.846655.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2072215.893674265,
|
| 9 |
-
"end_time": 2073130.940811355,
|
| 10 |
-
"total_evaluation_time_secondes": "915.047137090005",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.02799416848895062
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.48,
|
| 24 |
-
"qem_stderr": 0.07137140569598172
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.58,
|
| 28 |
-
"qem_stderr": 0.07050835816716033
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.3666666666666667,
|
| 32 |
-
"qem_stderr": 0.056624644117364224
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.3666666666666667,
|
| 36 |
-
"qem_stderr": 0.056624644117364224
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "1847a73ee97b84cf"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "07a33d50dbb61bc0"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "81be89d98a712a48"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "be0f2fadf1d23c89"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_mmos/results_2024-05-24T09-36-20.897469.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2117005.70480802,
|
| 9 |
-
"end_time": 2117317.858500111,
|
| 10 |
-
"total_evaluation_time_secondes": "312.15369209088385",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 19 |
-
"qem": 0.175,
|
| 20 |
-
"qem_stderr": 0.060843430844447585
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.175,
|
| 24 |
-
"qem_stderr": 0.060843430844447585
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_medium_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "3401efda8b0cbcb5",
|
| 66 |
-
"hash_full_prompts": "9553e2c8c7c87406",
|
| 67 |
-
"hash_input_tokens": "4f9c9b13fc9c2736",
|
| 68 |
-
"hash_cont_tokens": "dcf1a76b60e91789"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "4c81d27cfdb9d737",
|
| 81 |
-
"hash_full_prompts": "466bf00fd3e3d325",
|
| 82 |
-
"hash_input_tokens": "f3be8baebf6e5c0d",
|
| 83 |
-
"hash_cont_tokens": "4e52fc07aed5d27d"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_pot/results_2024-05-27T12-08-02.630027.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2108115.015177626,
|
| 9 |
-
"end_time": 2108455.320385662,
|
| 10 |
-
"total_evaluation_time_secondes": "340.3052080357447",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.175,
|
| 20 |
-
"qem_stderr": 0.06084343084444758
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.175,
|
| 24 |
-
"qem_stderr": 0.06084343084444758
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 33 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code_and_math"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "2799c24461029dc3",
|
| 66 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 67 |
-
"hash_input_tokens": "bb5427e5227cc6b4",
|
| 68 |
-
"hash_cont_tokens": "d8fff7d9380f6289"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "c72202b0a18ef5f9",
|
| 81 |
-
"hash_full_prompts": "a0429b0dea3e33db",
|
| 82 |
-
"hash_input_tokens": "8e6ef9684e80b71f",
|
| 83 |
-
"hash_cont_tokens": "4b3f922cf74d16d1"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_pot/results_2024-05-27T12-36-15.235885.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2106210.420319901,
|
| 9 |
-
"end_time": 2106551.181892837,
|
| 10 |
-
"total_evaluation_time_secondes": "340.7615729360841",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.175,
|
| 20 |
-
"qem_stderr": 0.06084343084444758
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.175,
|
| 24 |
-
"qem_stderr": 0.06084343084444758
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 33 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code_and_math"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "2799c24461029dc3",
|
| 66 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 67 |
-
"hash_input_tokens": "bb5427e5227cc6b4",
|
| 68 |
-
"hash_cont_tokens": "d8fff7d9380f6289"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "c72202b0a18ef5f9",
|
| 81 |
-
"hash_full_prompts": "a0429b0dea3e33db",
|
| 82 |
-
"hash_input_tokens": "8e6ef9684e80b71f",
|
| 83 |
-
"hash_cont_tokens": "4b3f922cf74d16d1"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.10/aimo_kaggle_medium_pot/results_2024-05-27T13-10-03.543903.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 9225308.278480008,
|
| 9 |
-
"end_time": 9225721.246559735,
|
| 10 |
-
"total_evaluation_time_secondes": "412.96807972714305",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "61a9aad6e4c0fc64852cc86f9381183b734be0e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.175,
|
| 20 |
-
"qem_stderr": 0.060843430844447585
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.15,
|
| 24 |
-
"qem_stderr": 0.05717718748968655
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.225,
|
| 28 |
-
"qem_stderr": 0.06686668711812965
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.18333333333333332,
|
| 32 |
-
"qem_stderr": 0.06162910181742126
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.18333333333333332,
|
| 36 |
-
"qem_stderr": 0.06162910181742126
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "7bb95d2c217c1b5c"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "6c0f57c20d743daf"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "8adfed93bcb46450"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "bd6abfe65da8351f"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_hard_mmos/results_2024-05-24T09-36-06.066222.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 1870349.343120157,
|
| 9 |
-
"end_time": 1870680.547112474,
|
| 10 |
-
"total_evaluation_time_secondes": "331.203992316965",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "0b525f62d709e3bb0fed36d4055acadaeffb2fea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_hard_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "b40b6a493a95bf77",
|
| 66 |
-
"hash_full_prompts": "bf4ae1359925ab3b",
|
| 67 |
-
"hash_input_tokens": "5f11181c1839f9c3",
|
| 68 |
-
"hash_cont_tokens": "747646d721b689b3"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "79dbebcff6acad9e",
|
| 81 |
-
"hash_full_prompts": "86a15c8410a494ef",
|
| 82 |
-
"hash_input_tokens": "28eff78f8cb1ac9f",
|
| 83 |
-
"hash_cont_tokens": "6df83d700244dcf8"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_hard_pot/results_2024-05-27T14-35-43.671086.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 342864.500085301,
|
| 9 |
-
"end_time": 343553.631159885,
|
| 10 |
-
"total_evaluation_time_secondes": "689.1310745839728",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "0b525f62d709e3bb0fed36d4055acadaeffb2fea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.02799416848895062
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.22,
|
| 24 |
-
"qem_stderr": 0.05917804336345136
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.5,
|
| 28 |
-
"qem_stderr": 0.07142857142857142
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.25333333333333335,
|
| 32 |
-
"qem_stderr": 0.05286692776032447
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.25333333333333335,
|
| 36 |
-
"qem_stderr": 0.05286692776032447
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "954b24cc2bdc4e6c"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "3a75a70b0409cc6d"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "277be9de6dc4234e"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "5297800052500dc2"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_medium_mmos/results_2024-05-24T09-36-32.170044.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2420257.809069017,
|
| 9 |
-
"end_time": 2420609.398094652,
|
| 10 |
-
"total_evaluation_time_secondes": "351.58902563527226",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "0b525f62d709e3bb0fed36d4055acadaeffb2fea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_medium_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "3401efda8b0cbcb5",
|
| 66 |
-
"hash_full_prompts": "9553e2c8c7c87406",
|
| 67 |
-
"hash_input_tokens": "4f9c9b13fc9c2736",
|
| 68 |
-
"hash_cont_tokens": "e0e2d443b31f87e8"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "4c81d27cfdb9d737",
|
| 81 |
-
"hash_full_prompts": "466bf00fd3e3d325",
|
| 82 |
-
"hash_input_tokens": "f3be8baebf6e5c0d",
|
| 83 |
-
"hash_cont_tokens": "67c15a3d33fd85be"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.11/aimo_kaggle_medium_pot/results_2024-05-27T14-31-11.406681.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 277940.525956996,
|
| 9 |
-
"end_time": 278355.570149596,
|
| 10 |
-
"total_evaluation_time_secondes": "415.0441926000058",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "0b525f62d709e3bb0fed36d4055acadaeffb2fea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.1,
|
| 20 |
-
"qem_stderr": 0.04803844614152612
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.1,
|
| 24 |
-
"qem_stderr": 0.04803844614152612
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.125,
|
| 28 |
-
"qem_stderr": 0.05295740910852021
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.10833333333333334,
|
| 32 |
-
"qem_stderr": 0.04967810046385748
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.10833333333333334,
|
| 36 |
-
"qem_stderr": 0.04967810046385748
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "8aca826f529dccef"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "2beb929a9d1f02a0"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "547712b40f6f642f"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "7494ce6ef42cbe4e"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_hard_mmos/results_2024-05-24T09-48-10.979290.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 1871072.599385947,
|
| 9 |
-
"end_time": 1871405.4602378,
|
| 10 |
-
"total_evaluation_time_secondes": "332.86085185292177",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "feac1ee8323888504974e4c56f4dd29dec9f5eea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_hard_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "b40b6a493a95bf77",
|
| 66 |
-
"hash_full_prompts": "bf4ae1359925ab3b",
|
| 67 |
-
"hash_input_tokens": "5f11181c1839f9c3",
|
| 68 |
-
"hash_cont_tokens": "6431505c900c5db1"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "79dbebcff6acad9e",
|
| 81 |
-
"hash_full_prompts": "86a15c8410a494ef",
|
| 82 |
-
"hash_input_tokens": "28eff78f8cb1ac9f",
|
| 83 |
-
"hash_cont_tokens": "6b37585ba81ea148"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_hard_pot/results_2024-05-27T14-35-53.533059.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 9230225.89268124,
|
| 9 |
-
"end_time": 9230871.207996167,
|
| 10 |
-
"total_evaluation_time_secondes": "645.3153149280697",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "feac1ee8323888504974e4c56f4dd29dec9f5eea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.02,
|
| 20 |
-
"qem_stderr": 0.02
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.06,
|
| 24 |
-
"qem_stderr": 0.033926691677251195
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.42,
|
| 28 |
-
"qem_stderr": 0.07050835816716035
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.16666666666666666,
|
| 32 |
-
"qem_stderr": 0.041478349948137185
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.16666666666666666,
|
| 36 |
-
"qem_stderr": 0.041478349948137185
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "dc39a3565f517db9"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "318061967e30b047"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "a2d575236416e3ca"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "9a548ef00ebb9273"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_medium_mmos/results_2024-05-24T09-48-12.415661.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2115893.229903704,
|
| 9 |
-
"end_time": 2116227.277975012,
|
| 10 |
-
"total_evaluation_time_secondes": "334.0480713080615",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "feac1ee8323888504974e4c56f4dd29dec9f5eea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_medium_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "3401efda8b0cbcb5",
|
| 66 |
-
"hash_full_prompts": "9553e2c8c7c87406",
|
| 67 |
-
"hash_input_tokens": "4f9c9b13fc9c2736",
|
| 68 |
-
"hash_cont_tokens": "22010b252d6c7aee"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "4c81d27cfdb9d737",
|
| 81 |
-
"hash_full_prompts": "466bf00fd3e3d325",
|
| 82 |
-
"hash_input_tokens": "f3be8baebf6e5c0d",
|
| 83 |
-
"hash_cont_tokens": "97d4e941c9cdda6b"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.12/aimo_kaggle_medium_pot/results_2024-05-27T14-30-36.611690.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 4057927.003502535,
|
| 9 |
-
"end_time": 4058278.547280695,
|
| 10 |
-
"total_evaluation_time_secondes": "351.54377815965563",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "feac1ee8323888504974e4c56f4dd29dec9f5eea",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.025,
|
| 20 |
-
"qem_stderr": 0.024999999999999998
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.15,
|
| 28 |
-
"qem_stderr": 0.05717718748968655
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.05833333333333333,
|
| 32 |
-
"qem_stderr": 0.027392395829895517
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.05833333333333333,
|
| 36 |
-
"qem_stderr": 0.027392395829895517
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "dea1ea4cd88ac25c"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "c30f828e6479e03f"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "4631fc712773916c"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "f61db1e864829049"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.13/aimo_kaggle_hard_mmos/results_2024-05-24T10-07-40.764237.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 1060.262766582,
|
| 9 |
-
"end_time": 1408.920575274,
|
| 10 |
-
"total_evaluation_time_secondes": "348.6578086919999",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "f2387bf0ced9308beb238e535b4d8334e7487ab2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_hard_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "b40b6a493a95bf77",
|
| 66 |
-
"hash_full_prompts": "bf4ae1359925ab3b",
|
| 67 |
-
"hash_input_tokens": "5f11181c1839f9c3",
|
| 68 |
-
"hash_cont_tokens": "4f6b08399af03ffd"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "79dbebcff6acad9e",
|
| 81 |
-
"hash_full_prompts": "86a15c8410a494ef",
|
| 82 |
-
"hash_input_tokens": "28eff78f8cb1ac9f",
|
| 83 |
-
"hash_cont_tokens": "575c9bad78c37155"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.13/aimo_kaggle_medium_mmos/results_2024-05-24T10-07-11.135359.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 1251998.579556463,
|
| 9 |
-
"end_time": 1252318.380801528,
|
| 10 |
-
"total_evaluation_time_secondes": "319.8012450649403",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "f2387bf0ced9308beb238e535b4d8334e7487ab2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_medium_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "3401efda8b0cbcb5",
|
| 66 |
-
"hash_full_prompts": "9553e2c8c7c87406",
|
| 67 |
-
"hash_input_tokens": "4f9c9b13fc9c2736",
|
| 68 |
-
"hash_cont_tokens": "cb917267759cd0c9"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "4c81d27cfdb9d737",
|
| 81 |
-
"hash_full_prompts": "466bf00fd3e3d325",
|
| 82 |
-
"hash_input_tokens": "f3be8baebf6e5c0d",
|
| 83 |
-
"hash_cont_tokens": "b39be9d73f59030c"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.14/aimo_kaggle_hard_mmos/results_2024-05-24T10-16-49.966984.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 1579.416964497,
|
| 9 |
-
"end_time": 1958.123144323,
|
| 10 |
-
"total_evaluation_time_secondes": "378.7061798259999",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "868636f68df95b6db35b4113753003b88836a86a",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 19 |
-
"qem": 0.02,
|
| 20 |
-
"qem_stderr": 0.02
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.02,
|
| 24 |
-
"qem_stderr": 0.02
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_hard_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "b40b6a493a95bf77",
|
| 66 |
-
"hash_full_prompts": "bf4ae1359925ab3b",
|
| 67 |
-
"hash_input_tokens": "5f11181c1839f9c3",
|
| 68 |
-
"hash_cont_tokens": "0592c8e0d76ec685"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "79dbebcff6acad9e",
|
| 81 |
-
"hash_full_prompts": "86a15c8410a494ef",
|
| 82 |
-
"hash_input_tokens": "28eff78f8cb1ac9f",
|
| 83 |
-
"hash_cont_tokens": "6cc5093a5dfe5f31"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.14/aimo_kaggle_medium_mmos/results_2024-05-24T10-16-38.488727.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 1252518.683488359,
|
| 9 |
-
"end_time": 1252885.734003984,
|
| 10 |
-
"total_evaluation_time_secondes": "367.05051562492736",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "868636f68df95b6db35b4113753003b88836a86a",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 19 |
-
"qem": 0.075,
|
| 20 |
-
"qem_stderr": 0.042176369614348695
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.075,
|
| 24 |
-
"qem_stderr": 0.042176369614348695
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_medium_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "3401efda8b0cbcb5",
|
| 66 |
-
"hash_full_prompts": "9553e2c8c7c87406",
|
| 67 |
-
"hash_input_tokens": "4f9c9b13fc9c2736",
|
| 68 |
-
"hash_cont_tokens": "5bd7b16ad1cce5f7"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "4c81d27cfdb9d737",
|
| 81 |
-
"hash_full_prompts": "466bf00fd3e3d325",
|
| 82 |
-
"hash_input_tokens": "f3be8baebf6e5c0d",
|
| 83 |
-
"hash_cont_tokens": "32fd13771d23980f"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.15/aimo_kaggle_hard_mmos/results_2024-05-24T10-25-09.540279.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 466.001152748,
|
| 9 |
-
"end_time": 788.580502107,
|
| 10 |
-
"total_evaluation_time_secondes": "322.57934935900005",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "c599676fdc00a821d271b74201c28a69b8dae2e7",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_hard_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_hard_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_hard_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 50,
|
| 56 |
-
"effective_num_docs": 50,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_hard_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "b40b6a493a95bf77",
|
| 66 |
-
"hash_full_prompts": "bf4ae1359925ab3b",
|
| 67 |
-
"hash_input_tokens": "5f11181c1839f9c3",
|
| 68 |
-
"hash_cont_tokens": "d220e4e743de8a9a"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 50,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 35,
|
| 73 |
-
"non_padded": 15,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "79dbebcff6acad9e",
|
| 81 |
-
"hash_full_prompts": "86a15c8410a494ef",
|
| 82 |
-
"hash_input_tokens": "28eff78f8cb1ac9f",
|
| 83 |
-
"hash_cont_tokens": "426a138c8b85ec49"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 50,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 35,
|
| 88 |
-
"non_padded": 15,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v01.15/aimo_kaggle_medium_mmos/results_2024-05-24T10-25-14.707607.json
DELETED
|
@@ -1,91 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2423204.564062136,
|
| 9 |
-
"end_time": 2423531.935635801,
|
| 10 |
-
"total_evaluation_time_secondes": "327.37157366471365",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "c599676fdc00a821d271b74201c28a69b8dae2e7",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"all": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
}
|
| 26 |
-
},
|
| 27 |
-
"versions": {
|
| 28 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": 0
|
| 29 |
-
},
|
| 30 |
-
"config_tasks": {
|
| 31 |
-
"custom|aimo_kaggle_medium_mmos:v0": {
|
| 32 |
-
"name": "aimo_kaggle_medium_mmos:v0",
|
| 33 |
-
"prompt_function": "mmos_medium_prompt_fn",
|
| 34 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 35 |
-
"hf_subset": "v0",
|
| 36 |
-
"metric": [
|
| 37 |
-
"quasi_exact_match_code"
|
| 38 |
-
],
|
| 39 |
-
"hf_avail_splits": [
|
| 40 |
-
"train"
|
| 41 |
-
],
|
| 42 |
-
"evaluation_splits": [
|
| 43 |
-
"train"
|
| 44 |
-
],
|
| 45 |
-
"few_shots_split": null,
|
| 46 |
-
"few_shots_select": null,
|
| 47 |
-
"generation_size": 2048,
|
| 48 |
-
"stop_sequence": null,
|
| 49 |
-
"output_regex": null,
|
| 50 |
-
"num_samples": null,
|
| 51 |
-
"frozen": false,
|
| 52 |
-
"suite": [
|
| 53 |
-
"custom"
|
| 54 |
-
],
|
| 55 |
-
"original_num_docs": 40,
|
| 56 |
-
"effective_num_docs": 40,
|
| 57 |
-
"trust_dataset": null,
|
| 58 |
-
"must_remove_duplicate_docs": null,
|
| 59 |
-
"version": 0
|
| 60 |
-
}
|
| 61 |
-
},
|
| 62 |
-
"summary_tasks": {
|
| 63 |
-
"custom|aimo_kaggle_medium_mmos:v0|0": {
|
| 64 |
-
"hashes": {
|
| 65 |
-
"hash_examples": "3401efda8b0cbcb5",
|
| 66 |
-
"hash_full_prompts": "9553e2c8c7c87406",
|
| 67 |
-
"hash_input_tokens": "4f9c9b13fc9c2736",
|
| 68 |
-
"hash_cont_tokens": "b3bfc1c6165ee81c"
|
| 69 |
-
},
|
| 70 |
-
"truncated": 40,
|
| 71 |
-
"non_truncated": 0,
|
| 72 |
-
"padded": 31,
|
| 73 |
-
"non_padded": 9,
|
| 74 |
-
"effective_few_shots": 0.0,
|
| 75 |
-
"num_truncated_few_shots": 0
|
| 76 |
-
}
|
| 77 |
-
},
|
| 78 |
-
"summary_general": {
|
| 79 |
-
"hashes": {
|
| 80 |
-
"hash_examples": "4c81d27cfdb9d737",
|
| 81 |
-
"hash_full_prompts": "466bf00fd3e3d325",
|
| 82 |
-
"hash_input_tokens": "f3be8baebf6e5c0d",
|
| 83 |
-
"hash_cont_tokens": "6e3f84e0c25bbd6c"
|
| 84 |
-
},
|
| 85 |
-
"truncated": 40,
|
| 86 |
-
"non_truncated": 0,
|
| 87 |
-
"padded": 31,
|
| 88 |
-
"non_padded": 9,
|
| 89 |
-
"num_truncated_few_shots": 0
|
| 90 |
-
}
|
| 91 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.01/aimo_kaggle_hard_pot/results_2024-05-27T22-52-32.480755.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 10719.748736187,
|
| 9 |
-
"end_time": 11433.49436362,
|
| 10 |
-
"total_evaluation_time_secondes": "713.7456274329998",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "ab83077ac83ffbbb8abe282b3dd8f4228cbbdcba",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.06,
|
| 20 |
-
"qem_stderr": 0.0339266916772512
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.38,
|
| 24 |
-
"qem_stderr": 0.06934092056863767
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.5,
|
| 28 |
-
"qem_stderr": 0.07142857142857142
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.3133333333333333,
|
| 32 |
-
"qem_stderr": 0.0582320612248201
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.3133333333333333,
|
| 36 |
-
"qem_stderr": 0.0582320612248201
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "c2831c3f8adfd2a5"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "e37a489b906d968e"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "43a13ab9e5e2cd6c"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "9ca954caaded26b5"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.01/aimo_kaggle_medium_pot/results_2024-05-27T22-49-32.235858.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 41425.043779027,
|
| 9 |
-
"end_time": 41783.252532695,
|
| 10 |
-
"total_evaluation_time_secondes": "358.20875366799737",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "ab83077ac83ffbbb8abe282b3dd8f4228cbbdcba",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.15,
|
| 20 |
-
"qem_stderr": 0.05717718748968655
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.225,
|
| 24 |
-
"qem_stderr": 0.06686668711812965
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.2,
|
| 28 |
-
"qem_stderr": 0.06405126152203487
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.19166666666666665,
|
| 32 |
-
"qem_stderr": 0.06269837870995036
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.19166666666666665,
|
| 36 |
-
"qem_stderr": 0.06269837870995036
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "129f73f798992c2d"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "c213703222b7b9d6"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "710c61a10724eb58"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "46841f1d968bb396"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.02/aimo_kaggle_hard_pot/results_2024-05-27T23-01-26.617019.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 9260481.786180781,
|
| 9 |
-
"end_time": 9261204.31955451,
|
| 10 |
-
"total_evaluation_time_secondes": "722.5333737283945",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "c579155dded92356530d24203be65dd06b745a34",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.06,
|
| 20 |
-
"qem_stderr": 0.0339266916772512
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.36,
|
| 24 |
-
"qem_stderr": 0.06857142857142856
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.5,
|
| 28 |
-
"qem_stderr": 0.07142857142857142
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.30666666666666664,
|
| 32 |
-
"qem_stderr": 0.057975563892417065
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.30666666666666664,
|
| 36 |
-
"qem_stderr": 0.057975563892417065
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "9aabf9d4ff47b254"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "9bf8175c55528051"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "3f123f6b38b6ffd5"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "afa8c82042b153d3"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.02/aimo_kaggle_medium_pot/results_2024-05-27T22-56-40.113739.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2107342.128751614,
|
| 9 |
-
"end_time": 2107777.207887237,
|
| 10 |
-
"total_evaluation_time_secondes": "435.0791356228292",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "c579155dded92356530d24203be65dd06b745a34",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.15,
|
| 20 |
-
"qem_stderr": 0.05717718748968655
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.15,
|
| 24 |
-
"qem_stderr": 0.05717718748968655
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.15,
|
| 28 |
-
"qem_stderr": 0.05717718748968655
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.15,
|
| 32 |
-
"qem_stderr": 0.05717718748968655
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.15,
|
| 36 |
-
"qem_stderr": 0.05717718748968655
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "9ed765ebd58d3013"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "2f085c109bc10f7d"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "1aedc4a3ff7a75a1"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "70d3921740f7ab02"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.03/aimo_kaggle_hard_pot/results_2024-05-27T22-50-33.482970.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 14259.254878757,
|
| 9 |
-
"end_time": 14914.427603002,
|
| 10 |
-
"total_evaluation_time_secondes": "655.1727242449997",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "8ccdf6776522c8613835892a71912fe2d8366885",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.02799416848895062
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.12,
|
| 24 |
-
"qem_stderr": 0.04642307659791979
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.54,
|
| 28 |
-
"qem_stderr": 0.07119963311072637
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.23333333333333336,
|
| 32 |
-
"qem_stderr": 0.048538959399198923
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.23333333333333336,
|
| 36 |
-
"qem_stderr": 0.048538959399198923
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "8fa9e18efe49021a"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "c0e55e5137310b5c"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "a1da9b9deee659aa"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "75cb29ad60e745ed"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.03/aimo_kaggle_medium_pot/results_2024-05-27T22-46-20.885308.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 106996.846626916,
|
| 9 |
-
"end_time": 107359.816964207,
|
| 10 |
-
"total_evaluation_time_secondes": "362.97033729100076",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "8ccdf6776522c8613835892a71912fe2d8366885",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.025,
|
| 20 |
-
"qem_stderr": 0.024999999999999994
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.05,
|
| 24 |
-
"qem_stderr": 0.03489912202260563
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.125,
|
| 28 |
-
"qem_stderr": 0.05295740910852021
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.06666666666666667,
|
| 32 |
-
"qem_stderr": 0.03761884371037528
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.06666666666666667,
|
| 36 |
-
"qem_stderr": 0.03761884371037528
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "398c838fac2a32cb"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "ad46c87b88dbba2f"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "246adbd385dd4579"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "2f97b0cecd2c9ec6"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.04/aimo_kaggle_hard_pot/results_2024-05-27T23-02-16.395548.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 304641.79705412,
|
| 9 |
-
"end_time": 305415.434942523,
|
| 10 |
-
"total_evaluation_time_secondes": "773.6378884030273",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "9d352e2a08647c995abd486f03c103c328f94b5e",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.08,
|
| 20 |
-
"qem_stderr": 0.03875617133214439
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.44,
|
| 24 |
-
"qem_stderr": 0.07091242083423345
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.38,
|
| 28 |
-
"qem_stderr": 0.06934092056863767
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.3,
|
| 32 |
-
"qem_stderr": 0.0596698375783385
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.3,
|
| 36 |
-
"qem_stderr": 0.0596698375783385
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "566a96752b9c4a94"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "7ab56adc56fc6b33"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "5c601ac7277feacc"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "3f013a410775284d"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.04/aimo_kaggle_medium_pot/results_2024-05-27T22-56-39.512900.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 18377.382677895,
|
| 9 |
-
"end_time": 18811.76337702,
|
| 10 |
-
"total_evaluation_time_secondes": "434.38069912500214",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "9d352e2a08647c995abd486f03c103c328f94b5e",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.15,
|
| 20 |
-
"qem_stderr": 0.05717718748968655
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.15,
|
| 24 |
-
"qem_stderr": 0.05717718748968655
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.175,
|
| 28 |
-
"qem_stderr": 0.060843430844447585
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.15833333333333333,
|
| 32 |
-
"qem_stderr": 0.05839926860794023
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.15833333333333333,
|
| 36 |
-
"qem_stderr": 0.05839926860794023
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "c0bcec869e50357e"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "175d7969de36ab1b"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "f3c3ab633ac76bd0"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "ed6a5a9ce37315ba"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.05/aimo_kaggle_hard_pot/results_2024-05-27T22-47-26.728707.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 211019.661163806,
|
| 9 |
-
"end_time": 211758.414953224,
|
| 10 |
-
"total_evaluation_time_secondes": "738.7537894179986",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "1b1c05ec5ff47d465eaaa801c306a11637d140a4",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.02799416848895062
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.36,
|
| 24 |
-
"qem_stderr": 0.06857142857142856
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.6,
|
| 28 |
-
"qem_stderr": 0.06998542122237653
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.3333333333333333,
|
| 32 |
-
"qem_stderr": 0.0555170060942519
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.3333333333333333,
|
| 36 |
-
"qem_stderr": 0.0555170060942519
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "6756ac049a892978"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "dfdc87594f75c999"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "63fc4eabc1edb20f"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "5cc4dd7edcddd58a"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.05/aimo_kaggle_medium_pot/results_2024-05-27T22-42-48.691321.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 40984.915370342,
|
| 9 |
-
"end_time": 41379.707945993,
|
| 10 |
-
"total_evaluation_time_secondes": "394.79257565100124",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "1b1c05ec5ff47d465eaaa801c306a11637d140a4",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.225,
|
| 20 |
-
"qem_stderr": 0.06686668711812965
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.2,
|
| 24 |
-
"qem_stderr": 0.06405126152203486
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.175,
|
| 28 |
-
"qem_stderr": 0.060843430844447585
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.20000000000000004,
|
| 32 |
-
"qem_stderr": 0.06392045982820403
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.20000000000000004,
|
| 36 |
-
"qem_stderr": 0.06392045982820403
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "a2da112f28b7b3dd"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "86ee58a3785c67b7"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "f4b14a0f9c0626a6"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "5b986fd2338ac693"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.06/aimo_kaggle_hard_pot/results_2024-05-27T23-22-45.715568.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 2382128.191168791,
|
| 9 |
-
"end_time": 2382914.608837258,
|
| 10 |
-
"total_evaluation_time_secondes": "786.4176684669219",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "86c0089055d54cd0922ae0d3b6a0455e83fe19a1",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.02799416848895062
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.36,
|
| 24 |
-
"qem_stderr": 0.06857142857142856
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.5,
|
| 28 |
-
"qem_stderr": 0.07142857142857142
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.3,
|
| 32 |
-
"qem_stderr": 0.055998056162983534
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.3,
|
| 36 |
-
"qem_stderr": 0.055998056162983534
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "96626341015b155a"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "944cb6439171b0d6"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "e20041a1a89e4258"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "81eae61419effeda"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.06/aimo_kaggle_medium_pot/results_2024-05-27T23-16-30.855912.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 44794.384758256,
|
| 9 |
-
"end_time": 45210.722487146,
|
| 10 |
-
"total_evaluation_time_secondes": "416.337728890001",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "86c0089055d54cd0922ae0d3b6a0455e83fe19a1",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.175,
|
| 20 |
-
"qem_stderr": 0.060843430844447585
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.125,
|
| 24 |
-
"qem_stderr": 0.05295740910852021
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.175,
|
| 28 |
-
"qem_stderr": 0.060843430844447564
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.15833333333333333,
|
| 32 |
-
"qem_stderr": 0.058214756932471794
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.15833333333333333,
|
| 36 |
-
"qem_stderr": 0.058214756932471794
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "f64111597c7c5f38"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "384432275bdfee30"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "5cd657852f93ead5"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "1d9394689bab0335"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.07/aimo_kaggle_hard_pot/results_2024-05-27T22-43-29.392522.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 15640.261235424,
|
| 9 |
-
"end_time": 16281.29244475,
|
| 10 |
-
"total_evaluation_time_secondes": "641.0312093260018",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "d572ef7608311c43ad723a3ac133e4503e7303a8",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.02,
|
| 20 |
-
"qem_stderr": 0.02
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.22,
|
| 24 |
-
"qem_stderr": 0.05917804336345138
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.56,
|
| 28 |
-
"qem_stderr": 0.07091242083423345
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.26666666666666666,
|
| 32 |
-
"qem_stderr": 0.050030154732561616
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.26666666666666666,
|
| 36 |
-
"qem_stderr": 0.050030154732561616
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "e8d46b691518bf31"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "96ef4d972ae45fd3"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "de4121a908cc86cf"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "974e7c9b71028589"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.07/aimo_kaggle_medium_pot/results_2024-05-27T22-39-34.119815.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 106589.039780829,
|
| 9 |
-
"end_time": 106953.051437736,
|
| 10 |
-
"total_evaluation_time_secondes": "364.0116569069942",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "d572ef7608311c43ad723a3ac133e4503e7303a8",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.1,
|
| 24 |
-
"qem_stderr": 0.04803844614152612
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.075,
|
| 28 |
-
"qem_stderr": 0.04217636961434867
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.05833333333333333,
|
| 32 |
-
"qem_stderr": 0.03007160525195826
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.05833333333333333,
|
| 36 |
-
"qem_stderr": 0.03007160525195826
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "af7a2a7c3bf735ce"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "3492201abfa537d3"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "b23881bd7634dfdf"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "37a711876b56979b"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.08/aimo_kaggle_hard_pot/results_2024-05-27T22-45-02.958180.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 3181770.247112947,
|
| 9 |
-
"end_time": 3182453.504957875,
|
| 10 |
-
"total_evaluation_time_secondes": "683.2578449277207",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "dc1193beda83f0cca3b7e60a158c36f6804af333",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.027994168488950606
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.1,
|
| 24 |
-
"qem_stderr": 0.04285714285714285
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.34,
|
| 28 |
-
"qem_stderr": 0.0676726816132972
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.16,
|
| 32 |
-
"qem_stderr": 0.04617466431979689
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.16,
|
| 36 |
-
"qem_stderr": 0.04617466431979689
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "aec6eb9abe870d9c"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "f5a8786338ef07ef"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "19cc51ae4f8a174c"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "0bced5d39b0648e2"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.08/aimo_kaggle_medium_pot/results_2024-05-27T22-39-55.232091.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 10322.250336088,
|
| 9 |
-
"end_time": 10676.245850647,
|
| 10 |
-
"total_evaluation_time_secondes": "353.9955145589993",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "dc1193beda83f0cca3b7e60a158c36f6804af333",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.0,
|
| 20 |
-
"qem_stderr": 0.0
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.0,
|
| 24 |
-
"qem_stderr": 0.0
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.1,
|
| 28 |
-
"qem_stderr": 0.048038446141526144
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.03333333333333333,
|
| 32 |
-
"qem_stderr": 0.016012815380508715
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.03333333333333333,
|
| 36 |
-
"qem_stderr": 0.016012815380508715
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "f6ef25c13e3b9beb"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "d2e29a088eb3ef40"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "05fe807135396e54"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "42dc377e96ff07f4"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.09/aimo_kaggle_hard_pot/results_2024-05-27T23-01-23.916419.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 45437.752892193,
|
| 9 |
-
"end_time": 46158.771446592,
|
| 10 |
-
"total_evaluation_time_secondes": "721.018554399001",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "4c5a6b0364eeff00e757545ad918edf9938087d7",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.02799416848895062
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.38,
|
| 24 |
-
"qem_stderr": 0.06934092056863767
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.5,
|
| 28 |
-
"qem_stderr": 0.07142857142857142
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.30666666666666664,
|
| 32 |
-
"qem_stderr": 0.05625455349538657
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.30666666666666664,
|
| 36 |
-
"qem_stderr": 0.05625455349538657
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "ee8d8deb951e1d20"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "4873c8bc9c937aea"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "0f9b2a78a3b11cef"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "28102952776d50f9"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.09/aimo_kaggle_medium_pot/results_2024-05-27T22-56-49.561125.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 43584.407888815,
|
| 9 |
-
"end_time": 44029.427719081,
|
| 10 |
-
"total_evaluation_time_secondes": "445.0198302659992",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "4c5a6b0364eeff00e757545ad918edf9938087d7",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.1,
|
| 20 |
-
"qem_stderr": 0.04803844614152612
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.1,
|
| 24 |
-
"qem_stderr": 0.04803844614152612
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.125,
|
| 28 |
-
"qem_stderr": 0.05295740910852021
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.10833333333333334,
|
| 32 |
-
"qem_stderr": 0.04967810046385748
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.10833333333333334,
|
| 36 |
-
"qem_stderr": 0.04967810046385748
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "fe53ab7e7343a382"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "afec954f4fad91f4"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "47fe73a6dd818fe2"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "8efd6f1fe2208bcb"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.10/aimo_kaggle_hard_pot/results_2024-05-27T22-56-26.205133.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 16333.701216751,
|
| 9 |
-
"end_time": 17058.105147357,
|
| 10 |
-
"total_evaluation_time_secondes": "724.4039306060004",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "18af21a11f19e82ae0b5c4807f6061cb2eb1458b",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.06,
|
| 20 |
-
"qem_stderr": 0.0339266916772512
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.44,
|
| 24 |
-
"qem_stderr": 0.07091242083423345
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.6,
|
| 28 |
-
"qem_stderr": 0.06998542122237653
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.3666666666666667,
|
| 32 |
-
"qem_stderr": 0.058274844577953726
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.3666666666666667,
|
| 36 |
-
"qem_stderr": 0.058274844577953726
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "28667310ce047385"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "20523b0a9736ca9a"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "fa8e50486f1f0438"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "f27ba9fc232dd906"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.10/aimo_kaggle_medium_pot/results_2024-05-27T22-52-23.993586.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 3182508.308928725,
|
| 9 |
-
"end_time": 3182894.568337251,
|
| 10 |
-
"total_evaluation_time_secondes": "386.25940852612257",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "18af21a11f19e82ae0b5c4807f6061cb2eb1458b",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.25,
|
| 20 |
-
"qem_stderr": 0.06933752452815363
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.15,
|
| 24 |
-
"qem_stderr": 0.05717718748968655
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.175,
|
| 28 |
-
"qem_stderr": 0.060843430844447585
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.19166666666666665,
|
| 32 |
-
"qem_stderr": 0.062452714287429247
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.19166666666666665,
|
| 36 |
-
"qem_stderr": 0.062452714287429247
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "33e1217843971331"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "0317f6719f9f3cd1"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "847c8e8d9dcacdd4"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "4d3a9d65f57acd85"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.11/aimo_kaggle_hard_pot/results_2024-05-27T23-28-10.769733.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 109144.315993371,
|
| 9 |
-
"end_time": 109869.701287677,
|
| 10 |
-
"total_evaluation_time_secondes": "725.3852943060047",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "5fbd6d2b74f2755ce0b32890432d6af62410c3e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.12,
|
| 20 |
-
"qem_stderr": 0.046423076597919784
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.34,
|
| 24 |
-
"qem_stderr": 0.0676726816132972
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.52,
|
| 28 |
-
"qem_stderr": 0.07137140569598172
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.32666666666666666,
|
| 32 |
-
"qem_stderr": 0.06182238796906623
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.32666666666666666,
|
| 36 |
-
"qem_stderr": 0.06182238796906623
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "e730032197dfa045"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "7b27c9de11d4cbd5"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "dc5b74898fd59f58"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "b759b18b73f3f7fe"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.11/aimo_kaggle_medium_pot/results_2024-05-27T23-24-01.746924.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 45275.122268848,
|
| 9 |
-
"end_time": 45661.613526662,
|
| 10 |
-
"total_evaluation_time_secondes": "386.49125781399925",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "5fbd6d2b74f2755ce0b32890432d6af62410c3e2",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.125,
|
| 20 |
-
"qem_stderr": 0.05295740910852021
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.125,
|
| 24 |
-
"qem_stderr": 0.05295740910852021
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.225,
|
| 28 |
-
"qem_stderr": 0.06686668711812965
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.15833333333333333,
|
| 32 |
-
"qem_stderr": 0.057593835111723356
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.15833333333333333,
|
| 36 |
-
"qem_stderr": 0.057593835111723356
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "fa3b910dfca55474"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "b1ae92916dde27d0"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "2febceafe1d903ce"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "4e6bfc37d6e77d6a"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.12/aimo_kaggle_hard_pot/results_2024-05-27T22-58-24.076261.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 107404.931245741,
|
| 9 |
-
"end_time": 108083.007317716,
|
| 10 |
-
"total_evaluation_time_secondes": "678.076071974996",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "651246d76fe6c8aedf46454ea474eca76136ab73",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 19 |
-
"qem": 0.04,
|
| 20 |
-
"qem_stderr": 0.02799416848895062
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 23 |
-
"qem": 0.3,
|
| 24 |
-
"qem_stderr": 0.06546536707079771
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 27 |
-
"qem": 0.52,
|
| 28 |
-
"qem_stderr": 0.07137140569598172
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_hard_pot:_average|0": {
|
| 31 |
-
"qem": 0.2866666666666667,
|
| 32 |
-
"qem_stderr": 0.05494364708524335
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.2866666666666667,
|
| 36 |
-
"qem_stderr": 0.05494364708524335
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_hard_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_hard_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_hard_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_hard_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_hard_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 50,
|
| 70 |
-
"effective_num_docs": 50,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_hard_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_hard_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 50,
|
| 100 |
-
"effective_num_docs": 50,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_hard_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_hard_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_hard_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-hard",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 50,
|
| 130 |
-
"effective_num_docs": 50,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_hard_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "303213a38d9f7512",
|
| 140 |
-
"hash_full_prompts": "3c670fa0e80bc301",
|
| 141 |
-
"hash_input_tokens": "1bd4187ad6963415",
|
| 142 |
-
"hash_cont_tokens": "713e9ea3fd7a21cd"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 50,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 37,
|
| 147 |
-
"non_padded": 13,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_hard_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "e4234b97ad92862f",
|
| 154 |
-
"hash_full_prompts": "09e7759a96f64e59",
|
| 155 |
-
"hash_input_tokens": "dc7f68cf14be3d61",
|
| 156 |
-
"hash_cont_tokens": "a24d886da74b19a1"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 50,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 32,
|
| 161 |
-
"non_padded": 18,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_hard_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "6396eb8833e13ba0",
|
| 168 |
-
"hash_full_prompts": "65f8b95ea99c7087",
|
| 169 |
-
"hash_input_tokens": "bddb67459e7601c5",
|
| 170 |
-
"hash_cont_tokens": "81abaeff7a78b548"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 50,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 30,
|
| 175 |
-
"non_padded": 20,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "648c9a107d279e1e",
|
| 183 |
-
"hash_full_prompts": "7f4c38eb08b3bb41",
|
| 184 |
-
"hash_input_tokens": "16f0fd76e0a2a165",
|
| 185 |
-
"hash_cont_tokens": "417fec7e4d72917f"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 150,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 99,
|
| 190 |
-
"non_padded": 51,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
eval_results/AI-MO/deepseek-math-7b-rl-sft/aimo_v02.12/aimo_kaggle_medium_pot/results_2024-05-27T22-54-41.166514.json
DELETED
|
@@ -1,193 +0,0 @@
|
|
| 1 |
-
{
|
| 2 |
-
"config_general": {
|
| 3 |
-
"lighteval_sha": "?",
|
| 4 |
-
"num_fewshot_seeds": 1,
|
| 5 |
-
"override_batch_size": 4,
|
| 6 |
-
"max_samples": null,
|
| 7 |
-
"job_id": "",
|
| 8 |
-
"start_time": 211814.762291854,
|
| 9 |
-
"end_time": 212192.853241965,
|
| 10 |
-
"total_evaluation_time_secondes": "378.09095011101454",
|
| 11 |
-
"model_name": "AI-MO/deepseek-math-7b-rl-sft",
|
| 12 |
-
"model_sha": "651246d76fe6c8aedf46454ea474eca76136ab73",
|
| 13 |
-
"model_dtype": "torch.bfloat16",
|
| 14 |
-
"model_size": "12.93 GB",
|
| 15 |
-
"config": null
|
| 16 |
-
},
|
| 17 |
-
"results": {
|
| 18 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 19 |
-
"qem": 0.1,
|
| 20 |
-
"qem_stderr": 0.04803844614152612
|
| 21 |
-
},
|
| 22 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 23 |
-
"qem": 0.125,
|
| 24 |
-
"qem_stderr": 0.05295740910852021
|
| 25 |
-
},
|
| 26 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 27 |
-
"qem": 0.1,
|
| 28 |
-
"qem_stderr": 0.04803844614152612
|
| 29 |
-
},
|
| 30 |
-
"custom|aimo_kaggle_medium_pot:_average|0": {
|
| 31 |
-
"qem": 0.10833333333333334,
|
| 32 |
-
"qem_stderr": 0.04967810046385749
|
| 33 |
-
},
|
| 34 |
-
"all": {
|
| 35 |
-
"qem": 0.10833333333333334,
|
| 36 |
-
"qem_stderr": 0.04967810046385749
|
| 37 |
-
}
|
| 38 |
-
},
|
| 39 |
-
"versions": {
|
| 40 |
-
"custom|aimo_kaggle_medium_pot:v0|0": 0,
|
| 41 |
-
"custom|aimo_kaggle_medium_pot:v1|0": 0,
|
| 42 |
-
"custom|aimo_kaggle_medium_pot:v2|0": 0
|
| 43 |
-
},
|
| 44 |
-
"config_tasks": {
|
| 45 |
-
"custom|aimo_kaggle_medium_pot:v0": {
|
| 46 |
-
"name": "aimo_kaggle_medium_pot:v0",
|
| 47 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v0",
|
| 48 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 49 |
-
"hf_subset": "v0",
|
| 50 |
-
"metric": [
|
| 51 |
-
"quasi_exact_match_code_and_math"
|
| 52 |
-
],
|
| 53 |
-
"hf_avail_splits": [
|
| 54 |
-
"train"
|
| 55 |
-
],
|
| 56 |
-
"evaluation_splits": [
|
| 57 |
-
"train"
|
| 58 |
-
],
|
| 59 |
-
"few_shots_split": null,
|
| 60 |
-
"few_shots_select": null,
|
| 61 |
-
"generation_size": 2048,
|
| 62 |
-
"stop_sequence": null,
|
| 63 |
-
"output_regex": null,
|
| 64 |
-
"num_samples": null,
|
| 65 |
-
"frozen": false,
|
| 66 |
-
"suite": [
|
| 67 |
-
"custom"
|
| 68 |
-
],
|
| 69 |
-
"original_num_docs": 40,
|
| 70 |
-
"effective_num_docs": 40,
|
| 71 |
-
"trust_dataset": null,
|
| 72 |
-
"must_remove_duplicate_docs": null,
|
| 73 |
-
"version": 0
|
| 74 |
-
},
|
| 75 |
-
"custom|aimo_kaggle_medium_pot:v1": {
|
| 76 |
-
"name": "aimo_kaggle_medium_pot:v1",
|
| 77 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v1",
|
| 78 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 79 |
-
"hf_subset": "v0",
|
| 80 |
-
"metric": [
|
| 81 |
-
"quasi_exact_match_code_and_math"
|
| 82 |
-
],
|
| 83 |
-
"hf_avail_splits": [
|
| 84 |
-
"train"
|
| 85 |
-
],
|
| 86 |
-
"evaluation_splits": [
|
| 87 |
-
"train"
|
| 88 |
-
],
|
| 89 |
-
"few_shots_split": null,
|
| 90 |
-
"few_shots_select": null,
|
| 91 |
-
"generation_size": 2048,
|
| 92 |
-
"stop_sequence": null,
|
| 93 |
-
"output_regex": null,
|
| 94 |
-
"num_samples": null,
|
| 95 |
-
"frozen": false,
|
| 96 |
-
"suite": [
|
| 97 |
-
"custom"
|
| 98 |
-
],
|
| 99 |
-
"original_num_docs": 40,
|
| 100 |
-
"effective_num_docs": 40,
|
| 101 |
-
"trust_dataset": null,
|
| 102 |
-
"must_remove_duplicate_docs": null,
|
| 103 |
-
"version": 0
|
| 104 |
-
},
|
| 105 |
-
"custom|aimo_kaggle_medium_pot:v2": {
|
| 106 |
-
"name": "aimo_kaggle_medium_pot:v2",
|
| 107 |
-
"prompt_function": "kaggle_medium_pot_prompt_fn_v2",
|
| 108 |
-
"hf_repo": "AI-MO/kaggle-validation-set-medium",
|
| 109 |
-
"hf_subset": "v0",
|
| 110 |
-
"metric": [
|
| 111 |
-
"quasi_exact_match_code_and_math"
|
| 112 |
-
],
|
| 113 |
-
"hf_avail_splits": [
|
| 114 |
-
"train"
|
| 115 |
-
],
|
| 116 |
-
"evaluation_splits": [
|
| 117 |
-
"train"
|
| 118 |
-
],
|
| 119 |
-
"few_shots_split": null,
|
| 120 |
-
"few_shots_select": null,
|
| 121 |
-
"generation_size": 2048,
|
| 122 |
-
"stop_sequence": null,
|
| 123 |
-
"output_regex": null,
|
| 124 |
-
"num_samples": null,
|
| 125 |
-
"frozen": false,
|
| 126 |
-
"suite": [
|
| 127 |
-
"custom"
|
| 128 |
-
],
|
| 129 |
-
"original_num_docs": 40,
|
| 130 |
-
"effective_num_docs": 40,
|
| 131 |
-
"trust_dataset": null,
|
| 132 |
-
"must_remove_duplicate_docs": null,
|
| 133 |
-
"version": 0
|
| 134 |
-
}
|
| 135 |
-
},
|
| 136 |
-
"summary_tasks": {
|
| 137 |
-
"custom|aimo_kaggle_medium_pot:v0|0": {
|
| 138 |
-
"hashes": {
|
| 139 |
-
"hash_examples": "2799c24461029dc3",
|
| 140 |
-
"hash_full_prompts": "de6572b914d5bba6",
|
| 141 |
-
"hash_input_tokens": "5af82ee284ccce29",
|
| 142 |
-
"hash_cont_tokens": "dfc8b121255332df"
|
| 143 |
-
},
|
| 144 |
-
"truncated": 40,
|
| 145 |
-
"non_truncated": 0,
|
| 146 |
-
"padded": 29,
|
| 147 |
-
"non_padded": 11,
|
| 148 |
-
"effective_few_shots": 0.0,
|
| 149 |
-
"num_truncated_few_shots": 0
|
| 150 |
-
},
|
| 151 |
-
"custom|aimo_kaggle_medium_pot:v1|0": {
|
| 152 |
-
"hashes": {
|
| 153 |
-
"hash_examples": "806b2e2056b41f84",
|
| 154 |
-
"hash_full_prompts": "427d5de6e4d90df2",
|
| 155 |
-
"hash_input_tokens": "be3dcf9f8a2350d0",
|
| 156 |
-
"hash_cont_tokens": "5530d0bf94ec3456"
|
| 157 |
-
},
|
| 158 |
-
"truncated": 40,
|
| 159 |
-
"non_truncated": 0,
|
| 160 |
-
"padded": 26,
|
| 161 |
-
"non_padded": 14,
|
| 162 |
-
"effective_few_shots": 0.0,
|
| 163 |
-
"num_truncated_few_shots": 0
|
| 164 |
-
},
|
| 165 |
-
"custom|aimo_kaggle_medium_pot:v2|0": {
|
| 166 |
-
"hashes": {
|
| 167 |
-
"hash_examples": "d8534375acc5d427",
|
| 168 |
-
"hash_full_prompts": "6bb90c129d6d6123",
|
| 169 |
-
"hash_input_tokens": "22eeaedbeb3f56f0",
|
| 170 |
-
"hash_cont_tokens": "e03d999936114bfc"
|
| 171 |
-
},
|
| 172 |
-
"truncated": 40,
|
| 173 |
-
"non_truncated": 0,
|
| 174 |
-
"padded": 31,
|
| 175 |
-
"non_padded": 9,
|
| 176 |
-
"effective_few_shots": 0.0,
|
| 177 |
-
"num_truncated_few_shots": 0
|
| 178 |
-
}
|
| 179 |
-
},
|
| 180 |
-
"summary_general": {
|
| 181 |
-
"hashes": {
|
| 182 |
-
"hash_examples": "623505a45a4910c2",
|
| 183 |
-
"hash_full_prompts": "3aca0fad61a42921",
|
| 184 |
-
"hash_input_tokens": "c7d2c8bb76f8ecb1",
|
| 185 |
-
"hash_cont_tokens": "45672d8b1cdafda4"
|
| 186 |
-
},
|
| 187 |
-
"truncated": 120,
|
| 188 |
-
"non_truncated": 0,
|
| 189 |
-
"padded": 86,
|
| 190 |
-
"non_padded": 34,
|
| 191 |
-
"num_truncated_few_shots": 0
|
| 192 |
-
}
|
| 193 |
-
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|