{
  "config_general": {
    "lighteval_sha": "?",
    "num_fewshot_seeds": 1,
    "override_batch_size": 1,
    "max_samples": null,
    "job_id": "",
    "start_time": 5099355.528818711,
    "end_time": 5099640.235462377,
    "total_evaluation_time_secondes": "284.70664366614074",
    "model_name": "teknium/OpenHermes-2.5-Mistral-7B",
    "model_sha": "24c0bea14d53e6f67f1fbe2eca5bfe7cae389b33",
    "model_dtype": "torch.bfloat16",
    "model_size": "13.99 GB",
    "config": null
  },
  "results": {
    "lighteval|agieval:aqua-rat|0": {
      "acc": 0.2440944881889764,
      "acc_stderr": 0.02700551612696104,
      "acc_norm": 0.25196850393700787,
      "acc_norm_stderr": 0.027294353392553598
    },
    "lighteval|agieval:gaokao-biology|0": {
      "acc": 0.38095238095238093,
      "acc_stderr": 0.033591100467499906,
      "acc_norm": 0.38571428571428573,
      "acc_norm_stderr": 0.03367014035794127
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "acc": 0.28502415458937197,
      "acc_stderr": 0.031452336098736614,
      "acc_norm": 0.28502415458937197,
      "acc_norm_stderr": 0.031452336098736614
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "acc": 0.3861788617886179,
      "acc_stderr": 0.031105137609389325,
      "acc_norm": 0.36585365853658536,
      "acc_norm_stderr": 0.03077268594539317
    },
    "lighteval|agieval:gaokao-english|0": {
      "acc": 0.6895424836601307,
      "acc_stderr": 0.02649303322514589,
      "acc_norm": 0.6830065359477124,
      "acc_norm_stderr": 0.026643278474508748
    },
    "lighteval|agieval:gaokao-geography|0": {
      "acc": 0.4120603015075377,
      "acc_stderr": 0.03497954737400385,
      "acc_norm": 0.4472361809045226,
      "acc_norm_stderr": 0.035335047084973224
    },
    "lighteval|agieval:gaokao-history|0": {
      "acc": 0.4765957446808511,
      "acc_stderr": 0.032650194750335815,
      "acc_norm": 0.4340425531914894,
      "acc_norm_stderr": 0.03240038086792747
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "acc": 0.2849002849002849,
      "acc_stderr": 0.02412657767241174,
      "acc_norm": 0.2962962962962963,
      "acc_norm_stderr": 0.024407539882901112
    },
    "lighteval|agieval:gaokao-physics|0": {
      "acc": 0.295,
      "acc_stderr": 0.032328014206142675,
      "acc_norm": 0.39,
      "acc_norm_stderr": 0.03457567623250012
    },
    "lighteval|agieval:logiqa-en|0": {
      "acc": 0.4116743471582181,
      "acc_stderr": 0.01930319140812144,
      "acc_norm": 0.38863287250384027,
      "acc_norm_stderr": 0.019118954063492326
    },
    "lighteval|agieval:logiqa-zh|0": {
      "acc": 0.38095238095238093,
      "acc_stderr": 0.019047619047618966,
      "acc_norm": 0.39477726574500765,
      "acc_norm_stderr": 0.019172423152334177
    },
    "lighteval|agieval:lsat-ar|0": {
      "acc": 0.23043478260869565,
      "acc_stderr": 0.02782780752227616,
      "acc_norm": 0.2217391304347826,
      "acc_norm_stderr": 0.02745149660405891
    },
    "lighteval|agieval:lsat-lr|0": {
      "acc": 0.5647058823529412,
      "acc_stderr": 0.02197574787518572,
      "acc_norm": 0.4627450980392157,
      "acc_norm_stderr": 0.022100505922784033
    },
    "lighteval|agieval:lsat-rc|0": {
      "acc": 0.6171003717472119,
      "acc_stderr": 0.029692924865649527,
      "acc_norm": 0.4795539033457249,
      "acc_norm_stderr": 0.030516814372510862
    },
    "lighteval|agieval:sat-en|0": {
      "acc": 0.7524271844660194,
      "acc_stderr": 0.030144409872297457,
      "acc_norm": 0.6747572815533981,
      "acc_norm_stderr": 0.03271904737596389
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "acc": 0.42718446601941745,
      "acc_stderr": 0.03454921537431906,
      "acc_norm": 0.3640776699029126,
      "acc_norm_stderr": 0.033606410551427816
    },
    "lighteval|agieval:sat-math|0": {
      "acc": 0.36363636363636365,
      "acc_stderr": 0.03250605554246891,
      "acc_norm": 0.35909090909090907,
      "acc_norm_stderr": 0.032417413727526295
    },
    "lighteval|agieval:_average|0": {
      "acc": 0.42367438112996464,
      "acc_stderr": 0.02875167229638612,
      "acc_norm": 0.40497154704312127,
      "acc_norm_stderr": 0.029038500241619625
    },
    "all": {
      "acc": 0.42367438112996464,
      "acc_stderr": 0.02875167229638612,
      "acc_norm": 0.40497154704312127,
      "acc_norm_stderr": 0.029038500241619625
    }
  },
  "versions": {
    "lighteval|agieval:aqua-rat|0": 0,
    "lighteval|agieval:gaokao-biology|0": 0,
    "lighteval|agieval:gaokao-chemistry|0": 0,
    "lighteval|agieval:gaokao-chinese|0": 0,
    "lighteval|agieval:gaokao-english|0": 0,
    "lighteval|agieval:gaokao-geography|0": 0,
    "lighteval|agieval:gaokao-history|0": 0,
    "lighteval|agieval:gaokao-mathqa|0": 0,
    "lighteval|agieval:gaokao-physics|0": 0,
    "lighteval|agieval:logiqa-en|0": 0,
    "lighteval|agieval:logiqa-zh|0": 0,
    "lighteval|agieval:lsat-ar|0": 0,
    "lighteval|agieval:lsat-lr|0": 0,
    "lighteval|agieval:lsat-rc|0": 0,
    "lighteval|agieval:sat-en|0": 0,
    "lighteval|agieval:sat-en-without-passage|0": 0,
    "lighteval|agieval:sat-math|0": 0
  },
  "config_tasks": {
    "lighteval|agieval:aqua-rat": {
      "name": "agieval:aqua-rat",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-aqua-rat",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 254,
      "effective_num_docs": 254,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-biology": {
      "name": "agieval:gaokao-biology",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-biology",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 210,
      "effective_num_docs": 210,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chemistry": {
      "name": "agieval:gaokao-chemistry",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chemistry",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 207,
      "effective_num_docs": 207,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-chinese": {
      "name": "agieval:gaokao-chinese",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-chinese",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 246,
      "effective_num_docs": 246,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-english": {
      "name": "agieval:gaokao-english",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-english",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 306,
      "effective_num_docs": 306,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-geography": {
      "name": "agieval:gaokao-geography",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-geography",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 199,
      "effective_num_docs": 199,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-history": {
      "name": "agieval:gaokao-history",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-history",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 235,
      "effective_num_docs": 235,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-mathqa": {
      "name": "agieval:gaokao-mathqa",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-mathqa",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 351,
      "effective_num_docs": 351,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:gaokao-physics": {
      "name": "agieval:gaokao-physics",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-gaokao-physics",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 200,
      "effective_num_docs": 200,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-en": {
      "name": "agieval:logiqa-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:logiqa-zh": {
      "name": "agieval:logiqa-zh",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-logiqa-zh",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 651,
      "effective_num_docs": 651,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-ar": {
      "name": "agieval:lsat-ar",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-ar",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 230,
      "effective_num_docs": 230,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-lr": {
      "name": "agieval:lsat-lr",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-lr",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 510,
      "effective_num_docs": 510,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:lsat-rc": {
      "name": "agieval:lsat-rc",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-lsat-rc",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 269,
      "effective_num_docs": 269,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en": {
      "name": "agieval:sat-en",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-en-without-passage": {
      "name": "agieval:sat-en-without-passage",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-en-without-passage",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 206,
      "effective_num_docs": 206,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    },
    "lighteval|agieval:sat-math": {
      "name": "agieval:sat-math",
      "prompt_function": "agieval",
      "hf_repo": "dmayhem93/agieval-sat-math",
      "hf_subset": "default",
      "metric": [
        "loglikelihood_acc",
        "loglikelihood_acc_norm_nospace"
      ],
      "hf_avail_splits": [
        "test"
      ],
      "evaluation_splits": [
        "test"
      ],
      "few_shots_split": null,
      "few_shots_select": "random_sampling",
      "generation_size": 1,
      "stop_sequence": null,
      "output_regex": null,
      "frozen": false,
      "suite": [
        "lighteval"
      ],
      "original_num_docs": 220,
      "effective_num_docs": 220,
      "trust_dataset": true,
      "must_remove_duplicate_docs": null
    }
  },
  "summary_tasks": {
    "lighteval|agieval:aqua-rat|0": {
      "hashes": {
        "hash_examples": "f09607f69e5b7525",
        "hash_full_prompts": "f0af1499da980246",
        "hash_input_tokens": "01586a40d59a3028",
        "hash_cont_tokens": "a12c4ac8996ba11d"
      },
      "truncated": 0,
      "non_truncated": 254,
      "padded": 1270,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-biology|0": {
      "hashes": {
        "hash_examples": "f262eaf4a72db963",
        "hash_full_prompts": "4027de4b1cdd1c67",
        "hash_input_tokens": "3356b55c1770d4be",
        "hash_cont_tokens": "22b786cf7aa6d1a9"
      },
      "truncated": 0,
      "non_truncated": 210,
      "padded": 840,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chemistry|0": {
      "hashes": {
        "hash_examples": "47f2e649f58d9da5",
        "hash_full_prompts": "39de31ab927f9675",
        "hash_input_tokens": "be1a2c35ca21a256",
        "hash_cont_tokens": "318562bcb4103fc4"
      },
      "truncated": 0,
      "non_truncated": 207,
      "padded": 831,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-chinese|0": {
      "hashes": {
        "hash_examples": "1010b21fde4726ab",
        "hash_full_prompts": "3b4313f1bd85fd2e",
        "hash_input_tokens": "b9a6f600f1a44225",
        "hash_cont_tokens": "7b177add04591cdb"
      },
      "truncated": 0,
      "non_truncated": 246,
      "padded": 982,
      "non_padded": 2,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-english|0": {
      "hashes": {
        "hash_examples": "4864e492a350ae93",
        "hash_full_prompts": "547863254a606496",
        "hash_input_tokens": "b485aadfee63f77a",
        "hash_cont_tokens": "c9ca0addab2a9327"
      },
      "truncated": 0,
      "non_truncated": 306,
      "padded": 1224,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-geography|0": {
      "hashes": {
        "hash_examples": "ec3a021e37650e7d",
        "hash_full_prompts": "0a7cffbf555ab29e",
        "hash_input_tokens": "5bf408915575d7cb",
        "hash_cont_tokens": "e1bc87e81807da78"
      },
      "truncated": 0,
      "non_truncated": 199,
      "padded": 796,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-history|0": {
      "hashes": {
        "hash_examples": "b3fad1596f1ae1f9",
        "hash_full_prompts": "b8aca4146c3435af",
        "hash_input_tokens": "469990442dc541ac",
        "hash_cont_tokens": "b3c6c60f59b08db4"
      },
      "truncated": 0,
      "non_truncated": 235,
      "padded": 940,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-mathqa|0": {
      "hashes": {
        "hash_examples": "1d1088556861b0b0",
        "hash_full_prompts": "1441e196c635c040",
        "hash_input_tokens": "388782ae768ee70a",
        "hash_cont_tokens": "5d69ebf8391bf298"
      },
      "truncated": 0,
      "non_truncated": 351,
      "padded": 1404,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:gaokao-physics|0": {
      "hashes": {
        "hash_examples": "eb05f035c7bfca2f",
        "hash_full_prompts": "be15722274b1466d",
        "hash_input_tokens": "6b2a1c4c5dd03798",
        "hash_cont_tokens": "93b4c52fa838ace2"
      },
      "truncated": 0,
      "non_truncated": 200,
      "padded": 800,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-en|0": {
      "hashes": {
        "hash_examples": "0a688a45f69c21e0",
        "hash_full_prompts": "ca179e67bdc726a6",
        "hash_input_tokens": "88b9eb8dd84c5f92",
        "hash_cont_tokens": "2624c1243afac3f2"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2604,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:logiqa-zh|0": {
      "hashes": {
        "hash_examples": "620d6888b6012ea5",
        "hash_full_prompts": "55e305ed89c6e580",
        "hash_input_tokens": "cdf0a9b880d48886",
        "hash_cont_tokens": "725ca2b921b6f8fe"
      },
      "truncated": 0,
      "non_truncated": 651,
      "padded": 2603,
      "non_padded": 1,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-ar|0": {
      "hashes": {
        "hash_examples": "627c8f5ccd5da209",
        "hash_full_prompts": "59e010e22954d5b7",
        "hash_input_tokens": "7a5e560985b7a87a",
        "hash_cont_tokens": "23c097e1d431f2b8"
      },
      "truncated": 0,
      "non_truncated": 230,
      "padded": 1137,
      "non_padded": 13,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-lr|0": {
      "hashes": {
        "hash_examples": "794641c86de172f5",
        "hash_full_prompts": "efc3c1a3a1586d3e",
        "hash_input_tokens": "ea471d0cc4d212d8",
        "hash_cont_tokens": "b555f4319746d815"
      },
      "truncated": 0,
      "non_truncated": 510,
      "padded": 2532,
      "non_padded": 18,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:lsat-rc|0": {
      "hashes": {
        "hash_examples": "35981ed917ea01cf",
        "hash_full_prompts": "b80e2b86e1eb0cea",
        "hash_input_tokens": "698fdaec9f62ef4b",
        "hash_cont_tokens": "8c1c4fc8c9cabd97"
      },
      "truncated": 0,
      "non_truncated": 269,
      "padded": 1345,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en|0": {
      "hashes": {
        "hash_examples": "041c39c646536a1e",
        "hash_full_prompts": "4eb610121b313521",
        "hash_input_tokens": "21ba0f545d8e5547",
        "hash_cont_tokens": "4837f17aae6c95e0"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 821,
      "non_padded": 0,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-en-without-passage|0": {
      "hashes": {
        "hash_examples": "e4d9284367dff68f",
        "hash_full_prompts": "532ea18906ff2f4e",
        "hash_input_tokens": "5b22e8182489bfd3",
        "hash_cont_tokens": "4837f17aae6c95e0"
      },
      "truncated": 0,
      "non_truncated": 206,
      "padded": 817,
      "non_padded": 4,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    },
    "lighteval|agieval:sat-math|0": {
      "hashes": {
        "hash_examples": "01db7291603fc1a0",
        "hash_full_prompts": "1422fad2e0cca51f",
        "hash_input_tokens": "d158847b7d653fa6",
        "hash_cont_tokens": "d959ef83452da9fe"
      },
      "truncated": 0,
      "non_truncated": 220,
      "padded": 877,
      "non_padded": 3,
      "effective_few_shots": 0.0,
      "num_truncated_few_shots": 0
    }
  },
  "summary_general": {
    "hashes": {
      "hash_examples": "da3af66181f18ddf",
      "hash_full_prompts": "e89209d4ce68d63a",
      "hash_input_tokens": "61b07fdce46e8b55",
      "hash_cont_tokens": "b3bace8c3199f6d8"
    },
    "truncated": 0,
    "non_truncated": 5151,
    "padded": 21823,
    "non_padded": 41,
    "num_truncated_few_shots": 0
  }
}