{
  "config_general": {
    "lighteval_sha": "?",
    "num_fewshot_seeds": 1,
    "override_batch_size": 4,
    "max_samples": null,
    "job_id": "",
    "start_time": 113890.410646821,
    "end_time": 114685.798350712,
    "total_evaluation_time_secondes": "795.3877038909995",
    "model_name": "Qwen/Qwen1.5-0.5B-Chat",
    "model_sha": "f82bd3692de0283f4a4b31e06d164dd8467fb52e",
    "model_dtype": "torch.bfloat16",
    "model_size": "1.05 GB",
    "config": null
  },
  "results": {
    "custom|mini_math:level_1|0": {
      "qem": 0.0,
      "qem_stderr": 0.0
    },
    "custom|mini_math:level_2|0": {
      "qem": 0.0,
      "qem_stderr": 0.0
    },
    "custom|mini_math:level_3|0": {
      "qem": 0.0,
      "qem_stderr": 0.0
    },
    "custom|mini_math:level_4|0": {
      "qem": 0.0,
      "qem_stderr": 0.0
    },
    "custom|mini_math:level_5|0": {
      "qem": 0.0,
      "qem_stderr": 0.0
    },
    "custom|mini_math:_average|0": {
      "qem": 0.0,
      "qem_stderr": 0.0
    },
    "all": {
      "qem": 0.0,
      "qem_stderr": 0.0
    }
  },
  "versions": {
    "custom|mini_math:level_1|0": 0,
    "custom|mini_math:level_2|0": 0,
    "custom|mini_math:level_3|0": 0,
    "custom|mini_math:level_4|0": 0,
    "custom|mini_math:level_5|0": 0
  },
  "config_tasks": {
    "custom|mini_math:level_1": {
      "name": "mini_math:level_1",
      "prompt_function": "math",
      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
      "hf_subset": "Level 1",
      "metric": [
        "quasi_exact_match_math"
      ],
      "hf_avail_splits": [
        "train",
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": null,
      "generation_size": 2048,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "custom",
        "mini_math"
      ],
      "original_num_docs": 38,
      "effective_num_docs": 38,
      "trust_dataset": null,
      "must_remove_duplicate_docs": null
    },
    "custom|mini_math:level_2": {
      "name": "mini_math:level_2",
      "prompt_function": "math",
      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
      "hf_subset": "Level 2",
      "metric": [
        "quasi_exact_match_math"
      ],
      "hf_avail_splits": [
        "train",
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": null,
      "generation_size": 2048,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "custom",
        "mini_math"
      ],
      "original_num_docs": 71,
      "effective_num_docs": 71,
      "trust_dataset": null,
      "must_remove_duplicate_docs": null
    },
    "custom|mini_math:level_3": {
      "name": "mini_math:level_3",
      "prompt_function": "math",
      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
      "hf_subset": "Level 3",
      "metric": [
        "quasi_exact_match_math"
      ],
      "hf_avail_splits": [
        "train",
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": null,
      "generation_size": 2048,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "custom",
        "mini_math"
      ],
      "original_num_docs": 78,
      "effective_num_docs": 78,
      "trust_dataset": null,
      "must_remove_duplicate_docs": null
    },
    "custom|mini_math:level_4": {
      "name": "mini_math:level_4",
      "prompt_function": "math",
      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
      "hf_subset": "Level 4",
      "metric": [
        "quasi_exact_match_math"
      ],
      "hf_avail_splits": [
        "train",
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": null,
      "generation_size": 2048,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "custom",
        "mini_math"
      ],
      "original_num_docs": 109,
      "effective_num_docs": 109,
      "trust_dataset": null,
      "must_remove_duplicate_docs": null
    },
    "custom|mini_math:level_5": {
      "name": "mini_math:level_5",
      "prompt_function": "math",
      "hf_repo": "HuggingFaceH4/lighteval-mini-math",
      "hf_subset": "Level 5",
      "metric": [
        "quasi_exact_match_math"
      ],
      "hf_avail_splits": [
        "train",
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": null,
      "generation_size": 2048,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "custom",
        "mini_math"
      ],
      "original_num_docs": 107,
      "effective_num_docs": 107,
      "trust_dataset": null,
      "must_remove_duplicate_docs": null
    }
  },
  "summary_tasks": {
    "custom|mini_math:level_1|0": {
      "hashes": {
        "hash_examples": "781a25d9a2f29e87",
        "hash_full_prompts": "09ef7f6df57f47ef",
        "hash_input_tokens": "a52e14904d82f4f0",
        "hash_cont_tokens": "1378bece65ca5077"
      },
      "truncated": 38,
      "non_truncated": 0,
      "padded": 27,
      "non_padded": 11,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "custom|mini_math:level_2|0": {
      "hashes": {
        "hash_examples": "471a4a8568c70994",
        "hash_full_prompts": "71779330375b1791",
        "hash_input_tokens": "3882edc6f5609345",
        "hash_cont_tokens": "ff9806e2946dc746"
      },
      "truncated": 71,
      "non_truncated": 0,
      "padded": 24,
      "non_padded": 47,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "custom|mini_math:level_3|0": {
      "hashes": {
        "hash_examples": "aa6979a90c9a7776",
        "hash_full_prompts": "a9999d3932558466",
        "hash_input_tokens": "01ceca3580b9eb5e",
        "hash_cont_tokens": "fbf0bffb8a2612ef"
      },
      "truncated": 78,
      "non_truncated": 0,
      "padded": 36,
      "non_padded": 42,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "custom|mini_math:level_4|0": {
      "hashes": {
        "hash_examples": "2e9e3be067de6ec6",
        "hash_full_prompts": "30f669c1c4439175",
        "hash_input_tokens": "4a8097cf379ba41c",
        "hash_cont_tokens": "88e54955e7213376"
      },
      "truncated": 109,
      "non_truncated": 0,
      "padded": 38,
      "non_padded": 71,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "custom|mini_math:level_5|0": {
      "hashes": {
        "hash_examples": "18b402b510f9a746",
        "hash_full_prompts": "0c8f48ff502c1190",
        "hash_input_tokens": "b6ecf04a8c2e48b4",
        "hash_cont_tokens": "c603094defd72220"
      },
      "truncated": 106,
      "non_truncated": 1,
      "padded": 38,
      "non_padded": 69,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    }
  },
  "summary_general": {
    "hashes": {
      "hash_examples": "470ce01cab0383c9",
      "hash_full_prompts": "36a8a8499eb2fd53",
      "hash_input_tokens": "a651497058968421",
      "hash_cont_tokens": "16318eec6b11890e"
    },
    "truncated": 402,
    "non_truncated": 1,
    "padded": 163,
    "non_padded": 240,
    "num_truncated_few_shots": 0
  }
}