{
  "config_general": {
    "lighteval_sha": "?",
    "num_fewshot_seeds": 1,
    "override_batch_size": 4,
    "max_samples": null,
    "job_id": "",
    "start_time": 155101.916908373,
    "end_time": 158135.917280056,
    "total_evaluation_time_secondes": "3034.0003716829815",
    "model_name": "abhishek/autotrain-mixtral-8x7b-orpo-v1",
    "model_sha": "a8be37cf01ad767a0c71e0ba3af29c0b3ebcb559",
    "model_dtype": "torch.bfloat16",
    "model_size": "87.49 GB",
    "config": null
  },
  "results": {
    "lighteval|agieval:aqua-rat|0": {
      "acc": 0.28346456692913385,
      "acc_stderr": 0.02833400492130763,
      "acc_norm": 0.2637795275590551,
      "acc_norm_stderr": 0.027705387412897232
    },
    "lighteval|agieval:gaokao-biology|0": {
      "acc": 0.2619047619047619,
      "acc_stderr": 0.030412684459928764,
      "acc_norm": 0.2904761904761905,
      "acc_norm_stderr": 0.03140260048069879
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "acc": 0.23671497584541062,
      "acc_stderr": 0.029615742669460064,
      "acc_norm": 0.28502415458937197,
      "acc_norm_stderr": 0.031452336098736614
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "acc": 0.2967479674796748,
      "acc_stderr": 0.029185445861037915,
      "acc_norm": 0.2967479674796748,
      "acc_norm_stderr": 0.029185445861037915
    },
    "lighteval|agieval:gaokao-english|0": {
      "acc": 0.7581699346405228,
      "acc_stderr": 0.024518195641879334,
      "acc_norm": 0.7058823529411765,
      "acc_norm_stderr": 0.026090162504279053
    },
    "lighteval|agieval:gaokao-geography|0": {
      "acc": 0.4321608040201005,
      "acc_stderr": 0.035204872502584535,
      "acc_norm": 0.3869346733668342,
      "acc_norm_stderr": 0.034613029011343124
    },
    "lighteval|agieval:gaokao-history|0": {
      "acc": 0.4340425531914894,
      "acc_stderr": 0.03240038086792747,
      "acc_norm": 0.39574468085106385,
      "acc_norm_stderr": 0.03196758697835363
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "acc": 0.2905982905982906,
      "acc_stderr": 0.024269376594480002,
      "acc_norm": 0.2962962962962963,
      "acc_norm_stderr": 0.024407539882901102
    },
    "lighteval|agieval:gaokao-physics|0": {
      "acc": 0.28,
      "acc_stderr": 0.031828687164775826,
      "acc_norm": 0.315,
      "acc_norm_stderr": 0.032928657464464904
    },
    "lighteval|agieval:logiqa-en|0": {
      "acc": 0.42857142857142855,
      "acc_stderr": 0.019410463442478737,
      "acc_norm": 0.4039938556067588,
      "acc_norm_stderr": 0.019246690834000657
    },
    "lighteval|agieval:logiqa-zh|0": {
      "acc": 0.2857142857142857,
      "acc_stderr": 0.01771924779845829,
      "acc_norm": 0.3225806451612903,
      "acc_norm_stderr": 0.018335437421251717
    },
    "lighteval|agieval:lsat-ar|0": {
      "acc": 0.21304347826086956,
      "acc_stderr": 0.02705775438993618,
      "acc_norm": 0.1782608695652174,
      "acc_norm_stderr": 0.025291655246273914
    },
    "lighteval|agieval:lsat-lr|0": {
      "acc": 0.5117647058823529,
      "acc_stderr": 0.02215597466931114,
      "acc_norm": 0.43529411764705883,
      "acc_norm_stderr": 0.02197574787518572
    },
    "lighteval|agieval:lsat-rc|0": {
      "acc": 0.6133828996282528,
      "acc_stderr": 0.029746711725453,
      "acc_norm": 0.5315985130111525,
      "acc_norm_stderr": 0.030481309049990975
    },
    "lighteval|agieval:sat-en|0": {
      "acc": 0.7864077669902912,
      "acc_stderr": 0.02862461766771065,
      "acc_norm": 0.7281553398058253,
      "acc_norm_stderr": 0.031073880563247468
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "acc": 0.5145631067961165,
      "acc_stderr": 0.03490669905098906,
      "acc_norm": 0.44660194174757284,
      "acc_norm_stderr": 0.034721796582639484
    },
    "lighteval|agieval:sat-math|0": {
      "acc": 0.44545454545454544,
      "acc_stderr": 0.03358522134954388,
      "acc_norm": 0.35909090909090907,
      "acc_norm_stderr": 0.0324174137275263
    },
    "lighteval|agieval:_average|0": {
      "acc": 0.4160415336416192,
      "acc_stderr": 0.028175063575133087,
      "acc_norm": 0.3906742373644382,
      "acc_norm_stderr": 0.028429216293813443
    },
    "all": {
      "acc": 0.4160415336416192,
      "acc_stderr": 0.028175063575133087,
      "acc_norm": 0.3906742373644382,
      "acc_norm_stderr": 0.028429216293813443
    }
  },
  "versions": {
    "lighteval|agieval:aqua-rat|0": 0,
    "lighteval|agieval:gaokao-biology|0": 0,
    "lighteval|agieval:gaokao-chemistry|0": 0,
    "lighteval|agieval:gaokao-chinese|0": 0,
    "lighteval|agieval:gaokao-english|0": 0,
    "lighteval|agieval:gaokao-geography|0": 0,
    "lighteval|agieval:gaokao-history|0": 0,
    "lighteval|agieval:gaokao-mathqa|0": 0,
    "lighteval|agieval:gaokao-physics|0": 0,
    "lighteval|agieval:logiqa-en|0": 0,
    "lighteval|agieval:logiqa-zh|0": 0,
    "lighteval|agieval:lsat-ar|0": 0,
    "lighteval|agieval:lsat-lr|0": 0,
    "lighteval|agieval:lsat-rc|0": 0,
    "lighteval|agieval:sat-en|0": 0,
    "lighteval|agieval:sat-en-without-passage|0": 0,
    "lighteval|agieval:sat-math|0": 0
  },
  "config_tasks": {
    "lighteval|agieval:aqua-rat": {
      "name": "agieval:aqua-rat",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-aqua-rat",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 254,
      "effective_num_docs": 254,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-biology": {
      "name": "agieval:gaokao-biology",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-biology",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 210,
      "effective_num_docs": 210,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chemistry": {
      "name": "agieval:gaokao-chemistry",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chemistry",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 207,
      "effective_num_docs": 207,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chinese": {
      "name": "agieval:gaokao-chinese",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chinese",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 246,
      "effective_num_docs": 246,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-english": {
      "name": "agieval:gaokao-english",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-english",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 306,
      "effective_num_docs": 306,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-geography": {
      "name": "agieval:gaokao-geography",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-geography",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 199,
      "effective_num_docs": 199,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-history": {
      "name": "agieval:gaokao-history",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-history",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 235,
      "effective_num_docs": 235,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-mathqa": {
      "name": "agieval:gaokao-mathqa",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-mathqa",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 351,
      "effective_num_docs": 351,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-physics": {
      "name": "agieval:gaokao-physics",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-physics",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 200,
      "effective_num_docs": 200,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-en": {
      "name": "agieval:logiqa-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-zh": {
      "name": "agieval:logiqa-zh",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-zh",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-ar": {
      "name": "agieval:lsat-ar",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-ar",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 230,
      "effective_num_docs": 230,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-lr": {
      "name": "agieval:lsat-lr",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-lr",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 510,
      "effective_num_docs": 510,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-rc": {
      "name": "agieval:lsat-rc",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-rc",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 269,
      "effective_num_docs": 269,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en": {
      "name": "agieval:sat-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en-without-passage": {
      "name": "agieval:sat-en-without-passage",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en-without-passage",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-math": {
      "name": "agieval:sat-math",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-math",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 220,
      "effective_num_docs": 220,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    }
  },
  "summary_tasks": {
    "lighteval|agieval:aqua-rat|0": {
      "hashes": {
        "hash_examples": "f09607f69e5b7525",
        "hash_full_prompts": "f0af1499da980246",
        "hash_input_tokens": "23ad8501912f9a35",
        "hash_cont_tokens": "5cb1512d1fe45a87"
      },
      "truncated": 0,
      "non_truncated": 254,
      "padded": 1270,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-biology|0": {
      "hashes": {
        "hash_examples": "f262eaf4a72db963",
        "hash_full_prompts": "4027de4b1cdd1c67",
        "hash_input_tokens": "dec771248a9b4d26",
        "hash_cont_tokens": "a469b12b65e9474e"
      },
      "truncated": 0,
      "non_truncated": 210,
      "padded": 840,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "hashes": {
        "hash_examples": "47f2e649f58d9da5",
        "hash_full_prompts": "39de31ab927f9675",
        "hash_input_tokens": "ac2e1e2f92e86b27",
        "hash_cont_tokens": "43337f7821e2165f"
      },
      "truncated": 0,
      "non_truncated": 207,
      "padded": 831,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "hashes": {
        "hash_examples": "1010b21fde4726ab",
        "hash_full_prompts": "3b4313f1bd85fd2e",
        "hash_input_tokens": "4337bd80a77f1b96",
        "hash_cont_tokens": "7c9a23bda503006a"
      },
      "truncated": 0,
      "non_truncated": 246,
      "padded": 982,
      "non_padded": 2,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-english|0": {
      "hashes": {
        "hash_examples": "4864e492a350ae93",
        "hash_full_prompts": "547863254a606496",
        "hash_input_tokens": "02dd45b62519e29a",
        "hash_cont_tokens": "2939fab73e305a01"
      },
      "truncated": 0,
      "non_truncated": 306,
      "padded": 1224,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-geography|0": {
      "hashes": {
        "hash_examples": "ec3a021e37650e7d",
        "hash_full_prompts": "0a7cffbf555ab29e",
        "hash_input_tokens": "4905dc7fbf95a714",
        "hash_cont_tokens": "b627d5a2a58e1b4e"
      },
      "truncated": 0,
      "non_truncated": 199,
      "padded": 796,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-history|0": {
      "hashes": {
        "hash_examples": "b3fad1596f1ae1f9",
        "hash_full_prompts": "b8aca4146c3435af",
        "hash_input_tokens": "45dd622494c7eba6",
        "hash_cont_tokens": "29af98bdef705752"
      },
      "truncated": 0,
      "non_truncated": 235,
      "padded": 940,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "hashes": {
        "hash_examples": "1d1088556861b0b0",
        "hash_full_prompts": "1441e196c635c040",
        "hash_input_tokens": "a6db369430f70502",
        "hash_cont_tokens": "e503c4178e232557"
      },
      "truncated": 0,
      "non_truncated": 351,
      "padded": 1404,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-physics|0": {
      "hashes": {
        "hash_examples": "eb05f035c7bfca2f",
        "hash_full_prompts": "be15722274b1466d",
        "hash_input_tokens": "20cf0004ad23a395",
        "hash_cont_tokens": "74730137f1150712"
      },
      "truncated": 0,
      "non_truncated": 200,
      "padded": 800,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-en|0": {
      "hashes": {
        "hash_examples": "0a688a45f69c21e0",
        "hash_full_prompts": "ca179e67bdc726a6",
        "hash_input_tokens": "8020ea34074302e8",
        "hash_cont_tokens": "406bb91c565edad6"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2604,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-zh|0": {
      "hashes": {
        "hash_examples": "620d6888b6012ea5",
        "hash_full_prompts": "55e305ed89c6e580",
        "hash_input_tokens": "b6b03fce18b14a92",
        "hash_cont_tokens": "9a57711e484917c2"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2603,
      "non_padded": 1,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-ar|0": {
      "hashes": {
        "hash_examples": "627c8f5ccd5da209",
        "hash_full_prompts": "59e010e22954d5b7",
        "hash_input_tokens": "f4504d090f5c0b3f",
        "hash_cont_tokens": "35b4b88913597c0b"
      },
      "truncated": 0,
      "non_truncated": 230,
      "padded": 1137,
      "non_padded": 13,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-lr|0": {
      "hashes": {
        "hash_examples": "794641c86de172f5",
        "hash_full_prompts": "efc3c1a3a1586d3e",
        "hash_input_tokens": "df779a462088fb9d",
        "hash_cont_tokens": "036ded82b481c7ce"
      },
      "truncated": 0,
      "non_truncated": 510,
      "padded": 2532,
      "non_padded": 18,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-rc|0": {
      "hashes": {
        "hash_examples": "35981ed917ea01cf",
        "hash_full_prompts": "b80e2b86e1eb0cea",
        "hash_input_tokens": "fa91b05e6e7ce596",
        "hash_cont_tokens": "800a77d64122b1b9"
      },
      "truncated": 0,
      "non_truncated": 269,
      "padded": 1345,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en|0": {
      "hashes": {
        "hash_examples": "041c39c646536a1e",
        "hash_full_prompts": "4eb610121b313521",
        "hash_input_tokens": "99e0785666d42b0c",
        "hash_cont_tokens": "70c3acb1407a2ef8"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 821,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "hashes": {
        "hash_examples": "e4d9284367dff68f",
        "hash_full_prompts": "532ea18906ff2f4e",
        "hash_input_tokens": "0070904e82d507eb",
        "hash_cont_tokens": "7f81dee0bb601ee9"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 817,
      "non_padded": 4,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-math|0": {
      "hashes": {
        "hash_examples": "01db7291603fc1a0",
        "hash_full_prompts": "1422fad2e0cca51f",
        "hash_input_tokens": "6b607d94aa2ca7d6",
        "hash_cont_tokens": "c555a90f08036a0f"
      },
      "truncated": 0,
      "non_truncated": 220,
      "padded": 877,
      "non_padded": 3,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    }
  },
  "summary_general": {
    "hashes": {
      "hash_examples": "da3af66181f18ddf",
      "hash_full_prompts": "e89209d4ce68d63a",
      "hash_input_tokens": "ca61573f55e495e6",
      "hash_cont_tokens": "832044d7eea021d0"
    },
    "truncated": 0,
    "non_truncated": 5151,
    "padded": 21823,
    "non_padded": 41,
    "num_truncated_few_shots": 0
  }
}