Upload eval_results/meta-llama/Llama-3.2-3B-Instruct/main/math/results_2024-10-06T16-58-55.587787.json with huggingface_hub

eval_results/meta-llama/Llama-3.2-3B-Instruct/main/math/results_2024-10-06T16-58-55.587787.json

@@ -0,0 +1,449 @@
+{
+  "config_general": {
+    "lighteval_sha": "?",
+    "num_fewshot_seeds": 1,
+    "override_batch_size": 4,
+    "max_samples": null,
+    "job_id": "",
+    "start_time": 468952.59722243,
+    "end_time": 507653.112541274,
+    "total_evaluation_time_secondes": "38700.515318844",
+    "model_name": "meta-llama/Llama-3.2-3B-Instruct",
+    "model_sha": "392a143b624368100f77a3eafaa4a2468ba50a72",
+    "model_dtype": "torch.bfloat16",
+    "model_size": "5.98 GB",
+    "config": null
+  },
+  "results": {
+    "lighteval|math:algebra|0": {
+      "qem": 0.13647851727042964,
+      "qem_stderr": 0.009968421116534627,
+      "maj@4": 0.11541701769165964,
+      "maj@4_stderr": 0.009278160257995039
+    },
+    "lighteval|math:counting_and_probability|0": {
+      "qem": 0.0759493670886076,
+      "qem_stderr": 0.012180900441921277,
+      "maj@4": 0.07172995780590717,
+      "maj@4_stderr": 0.011864703365865354
+    },
+    "lighteval|math:geometry|0": {
+      "qem": 0.11691022964509394,
+      "qem_stderr": 0.014696527629722107,
+      "maj@4": 0.11482254697286012,
+      "maj@4_stderr": 0.014581923359739
+    },
+    "lighteval|math:intermediate_algebra|0": {
+      "qem": 0.09523809523809523,
+      "qem_stderr": 0.009773930235808413,
+      "maj@4": 0.09523809523809523,
+      "maj@4_stderr": 0.009773930235808413
+    },
+    "lighteval|math:number_theory|0": {
+      "qem": 0.1111111111111111,
+      "qem_stderr": 0.01353655341940388,
+      "maj@4": 0.09074074074074075,
+      "maj@4_stderr": 0.012372305741867382
+    },
+    "lighteval|math:prealgebra|0": {
+      "qem": 0.08266360505166476,
+      "qem_stderr": 0.00933602178756277,
+      "maj@4": 0.07692307692307693,
+      "maj@4_stderr": 0.009034157223303146
+    },
+    "lighteval|math:precalculus|0": {
+      "qem": 0.1446886446886447,
+      "qem_stderr": 0.015068884082729252,
+      "maj@4": 0.14652014652014653,
+      "maj@4_stderr": 0.01514771264919227
+    },
+    "lighteval|math:_average|0": {
+      "qem": 0.10900565287052101,
+      "qem_stderr": 0.012080176959097475,
+      "maj@4": 0.10162736884178378,
+      "maj@4_stderr": 0.011721841833395799
+    },
+    "all": {
+      "qem": 0.10900565287052101,
+      "qem_stderr": 0.012080176959097475,
+      "maj@4": 0.10162736884178378,
+      "maj@4_stderr": 0.011721841833395799
+    }
+  },
+  "versions": {
+    "lighteval|math:algebra|0": 1,
+    "lighteval|math:counting_and_probability|0": 1,
+    "lighteval|math:geometry|0": 1,
+    "lighteval|math:intermediate_algebra|0": 1,
+    "lighteval|math:number_theory|0": 1,
+    "lighteval|math:prealgebra|0": 1,
+    "lighteval|math:precalculus|0": 1
+  },
+  "config_tasks": {
+    "lighteval|math:algebra": {
+      "name": "math:algebra",
+      "prompt_function": "math",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "algebra",
+      "metric": [
+        "quasi_exact_match_math",
+        "maj_at_4_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": [
+        "\n"
+      ],
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "lighteval",
+        "math"
+      ],
+      "original_num_docs": 1187,
+      "effective_num_docs": 1187,
+      "trust_dataset": true,
+      "must_remove_duplicate_docs": null,
+      "version": 1
+    },
+    "lighteval|math:counting_and_probability": {
+      "name": "math:counting_and_probability",
+      "prompt_function": "math",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "counting_and_probability",
+      "metric": [
+        "quasi_exact_match_math",
+        "maj_at_4_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": [
+        "\n"
+      ],
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "lighteval",
+        "math"
+      ],
+      "original_num_docs": 474,
+      "effective_num_docs": 474,
+      "trust_dataset": true,
+      "must_remove_duplicate_docs": null,
+      "version": 1
+    },
+    "lighteval|math:geometry": {
+      "name": "math:geometry",
+      "prompt_function": "math",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "geometry",
+      "metric": [
+        "quasi_exact_match_math",
+        "maj_at_4_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": [
+        "\n"
+      ],
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "lighteval",
+        "math"
+      ],
+      "original_num_docs": 479,
+      "effective_num_docs": 479,
+      "trust_dataset": true,
+      "must_remove_duplicate_docs": null,
+      "version": 1
+    },
+    "lighteval|math:intermediate_algebra": {
+      "name": "math:intermediate_algebra",
+      "prompt_function": "math",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "intermediate_algebra",
+      "metric": [
+        "quasi_exact_match_math",
+        "maj_at_4_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": [
+        "\n"
+      ],
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "lighteval",
+        "math"
+      ],
+      "original_num_docs": 903,
+      "effective_num_docs": 903,
+      "trust_dataset": true,
+      "must_remove_duplicate_docs": null,
+      "version": 1
+    },
+    "lighteval|math:number_theory": {
+      "name": "math:number_theory",
+      "prompt_function": "math",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "number_theory",
+      "metric": [
+        "quasi_exact_match_math",
+        "maj_at_4_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": [
+        "\n"
+      ],
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "lighteval",
+        "math"
+      ],
+      "original_num_docs": 540,
+      "effective_num_docs": 540,
+      "trust_dataset": true,
+      "must_remove_duplicate_docs": null,
+      "version": 1
+    },
+    "lighteval|math:prealgebra": {
+      "name": "math:prealgebra",
+      "prompt_function": "math",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "prealgebra",
+      "metric": [
+        "quasi_exact_match_math",
+        "maj_at_4_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": [
+        "\n"
+      ],
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "lighteval",
+        "math"
+      ],
+      "original_num_docs": 871,
+      "effective_num_docs": 871,
+      "trust_dataset": true,
+      "must_remove_duplicate_docs": null,
+      "version": 1
+    },
+    "lighteval|math:precalculus": {
+      "name": "math:precalculus",
+      "prompt_function": "math",
+      "hf_repo": "lighteval/MATH",
+      "hf_subset": "precalculus",
+      "metric": [
+        "quasi_exact_match_math",
+        "maj_at_4_math"
+      ],
+      "hf_avail_splits": [
+        "train",
+        "test",
+        "validation"
+      ],
+      "evaluation_splits": [
+        "test"
+      ],
+      "few_shots_split": null,
+      "few_shots_select": null,
+      "generation_size": 2048,
+      "stop_sequence": [
+        "\n"
+      ],
+      "output_regex": null,
+      "num_samples": null,
+      "frozen": false,
+      "suite": [
+        "lighteval",
+        "math"
+      ],
+      "original_num_docs": 546,
+      "effective_num_docs": 546,
+      "trust_dataset": true,
+      "must_remove_duplicate_docs": null,
+      "version": 1
+    }
+  },
+  "summary_tasks": {
+    "lighteval|math:algebra|0": {
+      "hashes": {
+        "hash_examples": "37a2fd2f076d2e49",
+        "hash_full_prompts": "d9a5b0f2a8114d54",
+        "hash_input_tokens": "eaf49802bcba393d",
+        "hash_cont_tokens": "48574d8fc6c82a2b"
+      },
+      "truncated": 1187,
+      "non_truncated": 0,
+      "padded": 173,
+      "non_padded": 1014,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "lighteval|math:counting_and_probability|0": {
+      "hashes": {
+        "hash_examples": "97b4892e28bc078b",
+        "hash_full_prompts": "db0180839766d2fc",
+        "hash_input_tokens": "6c1ea8e80f2d3547",
+        "hash_cont_tokens": "bafac5edeae67b53"
+      },
+      "truncated": 474,
+      "non_truncated": 0,
+      "padded": 44,
+      "non_padded": 430,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "lighteval|math:geometry|0": {
+      "hashes": {
+        "hash_examples": "9e9f0228b8b3d093",
+        "hash_full_prompts": "9f1f8446cd4f819d",
+        "hash_input_tokens": "b4ef39515dc76bc7",
+        "hash_cont_tokens": "6d2c7491b49e1e29"
+      },
+      "truncated": 479,
+      "non_truncated": 0,
+      "padded": 109,
+      "non_padded": 370,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "lighteval|math:intermediate_algebra|0": {
+      "hashes": {
+        "hash_examples": "cfe73a8e28ae94de",
+        "hash_full_prompts": "d9e693a7c7e2f26c",
+        "hash_input_tokens": "4703312c9e092f68",
+        "hash_cont_tokens": "3ab75e92be2efdb4"
+      },
+      "truncated": 900,
+      "non_truncated": 3,
+      "padded": 50,
+      "non_padded": 853,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "lighteval|math:number_theory|0": {
+      "hashes": {
+        "hash_examples": "4ee5237cf144afac",
+        "hash_full_prompts": "f16e298dfa576420",
+        "hash_input_tokens": "7006511b5ea80ae6",
+        "hash_cont_tokens": "1469f6f5075b7f6a"
+      },
+      "truncated": 540,
+      "non_truncated": 0,
+      "padded": 5,
+      "non_padded": 535,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "lighteval|math:prealgebra|0": {
+      "hashes": {
+        "hash_examples": "3fb3afeb885f73d8",
+        "hash_full_prompts": "0042b8b6b9844f6e",
+        "hash_input_tokens": "15bd31882799759e",
+        "hash_cont_tokens": "086d00e684329152"
+      },
+      "truncated": 871,
+      "non_truncated": 0,
+      "padded": 39,
+      "non_padded": 832,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    },
+    "lighteval|math:precalculus|0": {
+      "hashes": {
+        "hash_examples": "753e25ab9ec4b46c",
+        "hash_full_prompts": "be85bdd6e29e020f",
+        "hash_input_tokens": "632369d53714e8b3",
+        "hash_cont_tokens": "18b685d68cb708c6"
+      },
+      "truncated": 546,
+      "non_truncated": 0,
+      "padded": 18,
+      "non_padded": 528,
+      "effective_few_shots": 0.0,
+      "num_truncated_few_shots": 0
+    }
+  },
+  "summary_general": {
+    "hashes": {
+      "hash_examples": "b76099aa9092a203",
+      "hash_full_prompts": "2f90e54fa88614f5",
+      "hash_input_tokens": "17311b592ccff4a5",
+      "hash_cont_tokens": "0d1f0bbf21b83adc"
+    },
+    "truncated": 4997,
+    "non_truncated": 3,
+    "padded": 438,
+    "non_padded": 4562,
+    "num_truncated_few_shots": 0
+  }
+}