{
  "config_general": {
    "lighteval_sha": "?",
    "num_fewshot_seeds": 1,
    "override_batch_size": 1,
    "max_samples": null,
    "job_id": "",
    "start_time": 65374.436854381,
    "end_time": 67031.382509317,
    "total_evaluation_time_secondes": "1656.9456549360038",
    "model_name": "NousResearch/Nous-Hermes-2-Yi-34B",
    "model_sha": "fcb0a8847e76aea14aba9aa44009d4418ad7c18f",
    "model_dtype": "torch.bfloat16",
    "model_size": "64.17 GB",
    "config": null
  },
  "results": {
    "lighteval|agieval:aqua-rat|0": {
      "acc": 0.33858267716535434,
      "acc_stderr": 0.029751570051166513,
      "acc_norm": 0.33070866141732286,
      "acc_norm_stderr": 0.029578090029714024
    },
    "lighteval|agieval:gaokao-biology|0": {
      "acc": 0.8285714285714286,
      "acc_stderr": 0.026069544122384436,
      "acc_norm": 0.6952380952380952,
      "acc_norm_stderr": 0.031840067304739414
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "acc": 0.5990338164251208,
      "acc_stderr": 0.034146479384992046,
      "acc_norm": 0.45893719806763283,
      "acc_norm_stderr": 0.03471897205940103
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "acc": 0.8414634146341463,
      "acc_stderr": 0.02333454446028325,
      "acc_norm": 0.8252032520325203,
      "acc_norm_stderr": 0.024264094987893572
    },
    "lighteval|agieval:gaokao-english|0": {
      "acc": 0.7679738562091504,
      "acc_stderr": 0.024170840879340853,
      "acc_norm": 0.7483660130718954,
      "acc_norm_stderr": 0.0248480182638752
    },
    "lighteval|agieval:gaokao-geography|0": {
      "acc": 0.8592964824120602,
      "acc_stderr": 0.0247110613728616,
      "acc_norm": 0.8391959798994975,
      "acc_norm_stderr": 0.026106433978056173
    },
    "lighteval|agieval:gaokao-history|0": {
      "acc": 0.8765957446808511,
      "acc_stderr": 0.021500908854600244,
      "acc_norm": 0.8212765957446808,
      "acc_norm_stderr": 0.025045373272050975
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "acc": 0.34472934472934474,
      "acc_stderr": 0.02540478153035765,
      "acc_norm": 0.3561253561253561,
      "acc_norm_stderr": 0.025595763424550506
    },
    "lighteval|agieval:gaokao-physics|0": {
      "acc": 0.635,
      "acc_stderr": 0.03412767927155775,
      "acc_norm": 0.565,
      "acc_norm_stderr": 0.035143281737144066
    },
    "lighteval|agieval:logiqa-en|0": {
      "acc": 0.5376344086021505,
      "acc_stderr": 0.01955598083959783,
      "acc_norm": 0.45314900153609833,
      "acc_norm_stderr": 0.01952532808521508
    },
    "lighteval|agieval:logiqa-zh|0": {
      "acc": 0.6282642089093702,
      "acc_stderr": 0.0189553439882288,
      "acc_norm": 0.565284178187404,
      "acc_norm_stderr": 0.019443724559250246
    },
    "lighteval|agieval:lsat-ar|0": {
      "acc": 0.23478260869565218,
      "acc_stderr": 0.028009647070930125,
      "acc_norm": 0.20869565217391303,
      "acc_norm_stderr": 0.026854108265439654
    },
    "lighteval|agieval:lsat-lr|0": {
      "acc": 0.7235294117647059,
      "acc_stderr": 0.019824108780753004,
      "acc_norm": 0.5666666666666667,
      "acc_norm_stderr": 0.021964230412067968
    },
    "lighteval|agieval:lsat-rc|0": {
      "acc": 0.7843866171003717,
      "acc_stderr": 0.025120920402008615,
      "acc_norm": 0.6617100371747212,
      "acc_norm_stderr": 0.02890087690898019
    },
    "lighteval|agieval:sat-en|0": {
      "acc": 0.8737864077669902,
      "acc_stderr": 0.023194166778093068,
      "acc_norm": 0.8203883495145631,
      "acc_norm_stderr": 0.026810200597436818
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "acc": 0.5485436893203883,
      "acc_stderr": 0.03475654072342857,
      "acc_norm": 0.4223300970873786,
      "acc_norm_stderr": 0.03449760586825818
    },
    "lighteval|agieval:sat-math|0": {
      "acc": 0.4727272727272727,
      "acc_stderr": 0.033736570021804295,
      "acc_norm": 0.35454545454545455,
      "acc_norm_stderr": 0.032325610470146336
    },
    "lighteval|agieval:_average|0": {
      "acc": 0.6408765523361387,
      "acc_stderr": 0.02625709932543463,
      "acc_norm": 0.5701659169696,
      "acc_norm_stderr": 0.02749775177789526
    },
    "all": {
      "acc": 0.6408765523361387,
      "acc_stderr": 0.02625709932543463,
      "acc_norm": 0.5701659169696,
      "acc_norm_stderr": 0.02749775177789526
    }
  },
  "versions": {
    "lighteval|agieval:aqua-rat|0": 0,
    "lighteval|agieval:gaokao-biology|0": 0,
    "lighteval|agieval:gaokao-chemistry|0": 0,
    "lighteval|agieval:gaokao-chinese|0": 0,
    "lighteval|agieval:gaokao-english|0": 0,
    "lighteval|agieval:gaokao-geography|0": 0,
    "lighteval|agieval:gaokao-history|0": 0,
    "lighteval|agieval:gaokao-mathqa|0": 0,
    "lighteval|agieval:gaokao-physics|0": 0,
    "lighteval|agieval:logiqa-en|0": 0,
    "lighteval|agieval:logiqa-zh|0": 0,
    "lighteval|agieval:lsat-ar|0": 0,
    "lighteval|agieval:lsat-lr|0": 0,
    "lighteval|agieval:lsat-rc|0": 0,
    "lighteval|agieval:sat-en|0": 0,
    "lighteval|agieval:sat-en-without-passage|0": 0,
    "lighteval|agieval:sat-math|0": 0
  },
  "config_tasks": {
    "lighteval|agieval:aqua-rat": {
      "name": "agieval:aqua-rat",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-aqua-rat",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 254,
      "effective_num_docs": 254,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-biology": {
      "name": "agieval:gaokao-biology",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-biology",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 210,
      "effective_num_docs": 210,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chemistry": {
      "name": "agieval:gaokao-chemistry",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chemistry",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 207,
      "effective_num_docs": 207,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chinese": {
      "name": "agieval:gaokao-chinese",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chinese",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 246,
      "effective_num_docs": 246,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-english": {
      "name": "agieval:gaokao-english",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-english",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 306,
      "effective_num_docs": 306,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-geography": {
      "name": "agieval:gaokao-geography",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-geography",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 199,
      "effective_num_docs": 199,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-history": {
      "name": "agieval:gaokao-history",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-history",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 235,
      "effective_num_docs": 235,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-mathqa": {
      "name": "agieval:gaokao-mathqa",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-mathqa",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 351,
      "effective_num_docs": 351,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-physics": {
      "name": "agieval:gaokao-physics",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-physics",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 200,
      "effective_num_docs": 200,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-en": {
      "name": "agieval:logiqa-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-zh": {
      "name": "agieval:logiqa-zh",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-zh",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-ar": {
      "name": "agieval:lsat-ar",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-ar",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 230,
      "effective_num_docs": 230,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-lr": {
      "name": "agieval:lsat-lr",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-lr",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 510,
      "effective_num_docs": 510,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-rc": {
      "name": "agieval:lsat-rc",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-rc",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 269,
      "effective_num_docs": 269,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en": {
      "name": "agieval:sat-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en-without-passage": {
      "name": "agieval:sat-en-without-passage",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en-without-passage",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-math": {
      "name": "agieval:sat-math",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-math",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 220,
      "effective_num_docs": 220,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    }
  },
  "summary_tasks": {
    "lighteval|agieval:aqua-rat|0": {
      "hashes": {
        "hash_examples": "f09607f69e5b7525",
        "hash_full_prompts": "f0af1499da980246",
        "hash_input_tokens": "2d80b05e793655f3",
        "hash_cont_tokens": "6abbc14ca65d11e7"
      },
      "truncated": 0,
      "non_truncated": 254,
      "padded": 1270,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-biology|0": {
      "hashes": {
        "hash_examples": "f262eaf4a72db963",
        "hash_full_prompts": "4027de4b1cdd1c67",
        "hash_input_tokens": "145dad488c92a664",
        "hash_cont_tokens": "1f9a58017eff46b4"
      },
      "truncated": 0,
      "non_truncated": 210,
      "padded": 840,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "hashes": {
        "hash_examples": "47f2e649f58d9da5",
        "hash_full_prompts": "39de31ab927f9675",
        "hash_input_tokens": "f5b7cd7b8053bb67",
        "hash_cont_tokens": "ddc8619e56beeb14"
      },
      "truncated": 0,
      "non_truncated": 207,
      "padded": 831,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "hashes": {
        "hash_examples": "1010b21fde4726ab",
        "hash_full_prompts": "3b4313f1bd85fd2e",
        "hash_input_tokens": "11675c55d6846905",
        "hash_cont_tokens": "8dd5393ffc54add7"
      },
      "truncated": 0,
      "non_truncated": 246,
      "padded": 978,
      "non_padded": 6,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-english|0": {
      "hashes": {
        "hash_examples": "4864e492a350ae93",
        "hash_full_prompts": "547863254a606496",
        "hash_input_tokens": "704d436ae803852e",
        "hash_cont_tokens": "29c228a9a9bdae69"
      },
      "truncated": 0,
      "non_truncated": 306,
      "padded": 1224,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-geography|0": {
      "hashes": {
        "hash_examples": "ec3a021e37650e7d",
        "hash_full_prompts": "0a7cffbf555ab29e",
        "hash_input_tokens": "9014f62399a3994e",
        "hash_cont_tokens": "a695cb9ae7cce4af"
      },
      "truncated": 0,
      "non_truncated": 199,
      "padded": 796,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-history|0": {
      "hashes": {
        "hash_examples": "b3fad1596f1ae1f9",
        "hash_full_prompts": "b8aca4146c3435af",
        "hash_input_tokens": "5873ea11231f4c62",
        "hash_cont_tokens": "0e40bd675c0f473a"
      },
      "truncated": 0,
      "non_truncated": 235,
      "padded": 940,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "hashes": {
        "hash_examples": "1d1088556861b0b0",
        "hash_full_prompts": "1441e196c635c040",
        "hash_input_tokens": "d221b0d810969efe",
        "hash_cont_tokens": "5930c29fb0280970"
      },
      "truncated": 0,
      "non_truncated": 351,
      "padded": 1404,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-physics|0": {
      "hashes": {
        "hash_examples": "eb05f035c7bfca2f",
        "hash_full_prompts": "be15722274b1466d",
        "hash_input_tokens": "a42b73701059721f",
        "hash_cont_tokens": "f1c3497b8ee8d475"
      },
      "truncated": 0,
      "non_truncated": 200,
      "padded": 800,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-en|0": {
      "hashes": {
        "hash_examples": "0a688a45f69c21e0",
        "hash_full_prompts": "ca179e67bdc726a6",
        "hash_input_tokens": "47fb91c9a3344ba7",
        "hash_cont_tokens": "fbb2e8cb63b80f82"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2599,
      "non_padded": 5,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-zh|0": {
      "hashes": {
        "hash_examples": "620d6888b6012ea5",
        "hash_full_prompts": "55e305ed89c6e580",
        "hash_input_tokens": "b5813a1a83314763",
        "hash_cont_tokens": "82ee29b2aaf17f35"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2600,
      "non_padded": 4,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-ar|0": {
      "hashes": {
        "hash_examples": "627c8f5ccd5da209",
        "hash_full_prompts": "59e010e22954d5b7",
        "hash_input_tokens": "f03de938aac8ea62",
        "hash_cont_tokens": "290ac06e51cb7252"
      },
      "truncated": 0,
      "non_truncated": 230,
      "padded": 1128,
      "non_padded": 22,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-lr|0": {
      "hashes": {
        "hash_examples": "794641c86de172f5",
        "hash_full_prompts": "efc3c1a3a1586d3e",
        "hash_input_tokens": "888b77a37afd6b82",
        "hash_cont_tokens": "05f1609f054e144f"
      },
      "truncated": 0,
      "non_truncated": 510,
      "padded": 2526,
      "non_padded": 24,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-rc|0": {
      "hashes": {
        "hash_examples": "35981ed917ea01cf",
        "hash_full_prompts": "b80e2b86e1eb0cea",
        "hash_input_tokens": "b752432568c70c2b",
        "hash_cont_tokens": "05d7e4f3d88d7ac7"
      },
      "truncated": 0,
      "non_truncated": 269,
      "padded": 1345,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en|0": {
      "hashes": {
        "hash_examples": "041c39c646536a1e",
        "hash_full_prompts": "4eb610121b313521",
        "hash_input_tokens": "dacd7d386df9a412",
        "hash_cont_tokens": "bb9b8e94666339c2"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 821,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "hashes": {
        "hash_examples": "e4d9284367dff68f",
        "hash_full_prompts": "532ea18906ff2f4e",
        "hash_input_tokens": "40574516a39a0eed",
        "hash_cont_tokens": "bb9b8e94666339c2"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 808,
      "non_padded": 13,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-math|0": {
      "hashes": {
        "hash_examples": "01db7291603fc1a0",
        "hash_full_prompts": "1422fad2e0cca51f",
        "hash_input_tokens": "0974baa9ae5b0063",
        "hash_cont_tokens": "ec56de0852c3e68a"
      },
      "truncated": 0,
      "non_truncated": 220,
      "padded": 874,
      "non_padded": 6,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    }
  },
  "summary_general": {
    "hashes": {
      "hash_examples": "da3af66181f18ddf",
      "hash_full_prompts": "e89209d4ce68d63a",
      "hash_input_tokens": "9a19007309de5473",
      "hash_cont_tokens": "d990eb8795d1e87d"
    },
    "truncated": 0,
    "non_truncated": 5151,
    "padded": 21784,
    "non_padded": 80,
    "num_truncated_few_shots": 0
  }
}