{ "config_general": { "lighteval_sha": "?", "num_fewshot_seeds": 1, "override_batch_size": 1, "max_samples": null, "job_id": "", "start_time": 408094.258556013, "end_time": 411831.442471242, "total_evaluation_time_secondes": "3737.1839152289904", "model_name": "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO", "model_sha": "ba6df46252dff6f3978ed416fd95523a3aadb554", "model_dtype": "torch.bfloat16", "model_size": "87.49 GB", "config": null }, "results": { "lighteval|mmlu:abstract_algebra|5": { "acc": 0.51, "acc_stderr": 0.05024183937956911 }, "lighteval|mmlu:anatomy|5": { "acc": 0.7037037037037037, "acc_stderr": 0.03944624162501116 }, "lighteval|mmlu:astronomy|5": { "acc": 0.8092105263157895, "acc_stderr": 0.031975658210325 }, "lighteval|mmlu:business_ethics|5": { "acc": 0.75, "acc_stderr": 0.04351941398892446 }, "lighteval|mmlu:clinical_knowledge|5": { "acc": 0.7849056603773585, "acc_stderr": 0.025288394502891366 }, "lighteval|mmlu:college_biology|5": { "acc": 0.8333333333333334, "acc_stderr": 0.031164899666948614 }, "lighteval|mmlu:college_chemistry|5": { "acc": 0.54, "acc_stderr": 0.05009082659620332 }, "lighteval|mmlu:college_computer_science|5": { "acc": 0.62, "acc_stderr": 0.048783173121456316 }, "lighteval|mmlu:college_mathematics|5": { "acc": 0.5, "acc_stderr": 0.050251890762960605 }, "lighteval|mmlu:college_medicine|5": { "acc": 0.6820809248554913, "acc_stderr": 0.035506839891655796 }, "lighteval|mmlu:college_physics|5": { "acc": 0.5196078431372549, "acc_stderr": 0.04971358884367406 }, "lighteval|mmlu:computer_security|5": { "acc": 0.81, "acc_stderr": 0.039427724440366234 }, "lighteval|mmlu:conceptual_physics|5": { "acc": 0.6723404255319149, "acc_stderr": 0.030683020843231008 }, "lighteval|mmlu:econometrics|5": { "acc": 0.6491228070175439, "acc_stderr": 0.04489539350270698 }, "lighteval|mmlu:electrical_engineering|5": { "acc": 0.6689655172413793, "acc_stderr": 0.039215453124671215 }, "lighteval|mmlu:elementary_mathematics|5": { "acc": 0.49206349206349204, "acc_stderr": 0.02574806587167329 }, "lighteval|mmlu:formal_logic|5": { "acc": 0.5317460317460317, "acc_stderr": 0.04463112720677172 }, "lighteval|mmlu:global_facts|5": { "acc": 0.41, "acc_stderr": 0.049431107042371025 }, "lighteval|mmlu:high_school_biology|5": { "acc": 0.8516129032258064, "acc_stderr": 0.020222737554330378 }, "lighteval|mmlu:high_school_chemistry|5": { "acc": 0.6059113300492611, "acc_stderr": 0.03438157967036545 }, "lighteval|mmlu:high_school_computer_science|5": { "acc": 0.71, "acc_stderr": 0.045604802157206845 }, "lighteval|mmlu:high_school_european_history|5": { "acc": 0.8363636363636363, "acc_stderr": 0.02888787239548795 }, "lighteval|mmlu:high_school_geography|5": { "acc": 0.8585858585858586, "acc_stderr": 0.024825909793343336 }, "lighteval|mmlu:high_school_government_and_politics|5": { "acc": 0.9378238341968912, "acc_stderr": 0.017426974154240524 }, "lighteval|mmlu:high_school_macroeconomics|5": { "acc": 0.6871794871794872, "acc_stderr": 0.023507579020645358 }, "lighteval|mmlu:high_school_mathematics|5": { "acc": 0.34074074074074073, "acc_stderr": 0.028897748741131137 }, "lighteval|mmlu:high_school_microeconomics|5": { "acc": 0.8067226890756303, "acc_stderr": 0.025649470265889186 }, "lighteval|mmlu:high_school_physics|5": { "acc": 0.46357615894039733, "acc_stderr": 0.04071636065944215 }, "lighteval|mmlu:high_school_psychology|5": { "acc": 0.8880733944954129, "acc_stderr": 0.013517352714958785 }, "lighteval|mmlu:high_school_statistics|5": { "acc": 0.5972222222222222, "acc_stderr": 0.03344887382997866 }, "lighteval|mmlu:high_school_us_history|5": { "acc": 0.8480392156862745, "acc_stderr": 0.025195658428931792 }, "lighteval|mmlu:high_school_world_history|5": { "acc": 0.8945147679324894, "acc_stderr": 0.01999556072375854 }, "lighteval|mmlu:human_aging|5": { "acc": 0.7668161434977578, "acc_stderr": 0.028380391147094702 }, "lighteval|mmlu:human_sexuality|5": { "acc": 0.8702290076335878, "acc_stderr": 0.029473649496907065 }, "lighteval|mmlu:international_law|5": { "acc": 0.8512396694214877, "acc_stderr": 0.03248470083807194 }, "lighteval|mmlu:jurisprudence|5": { "acc": 0.8240740740740741, "acc_stderr": 0.036809181416738807 }, "lighteval|mmlu:logical_fallacies|5": { "acc": 0.7852760736196319, "acc_stderr": 0.03226219377286775 }, "lighteval|mmlu:machine_learning|5": { "acc": 0.5803571428571429, "acc_stderr": 0.046840993210771065 }, "lighteval|mmlu:management|5": { "acc": 0.8640776699029126, "acc_stderr": 0.03393295729761012 }, "lighteval|mmlu:marketing|5": { "acc": 0.9102564102564102, "acc_stderr": 0.018724301741941642 }, "lighteval|mmlu:medical_genetics|5": { "acc": 0.79, "acc_stderr": 0.040936018074033256 }, "lighteval|mmlu:miscellaneous|5": { "acc": 0.879948914431673, "acc_stderr": 0.011622736692041268 }, "lighteval|mmlu:moral_disputes|5": { "acc": 0.791907514450867, "acc_stderr": 0.021855255263421795 }, "lighteval|mmlu:moral_scenarios|5": { "acc": 0.547486033519553, "acc_stderr": 0.016646914804438785 }, "lighteval|mmlu:nutrition|5": { "acc": 0.7712418300653595, "acc_stderr": 0.02405102973991225 }, "lighteval|mmlu:philosophy|5": { "acc": 0.7684887459807074, "acc_stderr": 0.023956532766639133 }, "lighteval|mmlu:prehistory|5": { "acc": 0.8179012345679012, "acc_stderr": 0.021473491834808345 }, "lighteval|mmlu:professional_accounting|5": { "acc": 0.5602836879432624, "acc_stderr": 0.029609912075594113 }, "lighteval|mmlu:professional_law|5": { "acc": 0.5436766623207301, "acc_stderr": 0.012721420501462547 }, "lighteval|mmlu:professional_medicine|5": { "acc": 0.7941176470588235, "acc_stderr": 0.024562204314142314 }, "lighteval|mmlu:professional_psychology|5": { "acc": 0.7679738562091504, "acc_stderr": 0.017077373377856923 }, "lighteval|mmlu:public_relations|5": { "acc": 0.7181818181818181, "acc_stderr": 0.04309118709946458 }, "lighteval|mmlu:security_studies|5": { "acc": 0.8163265306122449, "acc_stderr": 0.02478907133200765 }, "lighteval|mmlu:sociology|5": { "acc": 0.8706467661691543, "acc_stderr": 0.023729830881018533 }, "lighteval|mmlu:us_foreign_policy|5": { "acc": 0.9, "acc_stderr": 0.030151134457776334 }, "lighteval|mmlu:virology|5": { "acc": 0.536144578313253, "acc_stderr": 0.038823108508905954 }, "lighteval|mmlu:world_religions|5": { "acc": 0.8771929824561403, "acc_stderr": 0.025172984350155754 }, "lighteval|mmlu:_average|5": { "acc": 0.719601605220369, "acc_stderr": 0.031604766907487786 } }, "versions": { "lighteval|mmlu:abstract_algebra|5": 0, "lighteval|mmlu:anatomy|5": 0, "lighteval|mmlu:astronomy|5": 0, "lighteval|mmlu:business_ethics|5": 0, "lighteval|mmlu:clinical_knowledge|5": 0, "lighteval|mmlu:college_biology|5": 0, "lighteval|mmlu:college_chemistry|5": 0, "lighteval|mmlu:college_computer_science|5": 0, "lighteval|mmlu:college_mathematics|5": 0, "lighteval|mmlu:college_medicine|5": 0, "lighteval|mmlu:college_physics|5": 0, "lighteval|mmlu:computer_security|5": 0, "lighteval|mmlu:conceptual_physics|5": 0, "lighteval|mmlu:econometrics|5": 0, "lighteval|mmlu:electrical_engineering|5": 0, "lighteval|mmlu:elementary_mathematics|5": 0, "lighteval|mmlu:formal_logic|5": 0, "lighteval|mmlu:global_facts|5": 0, "lighteval|mmlu:high_school_biology|5": 0, "lighteval|mmlu:high_school_chemistry|5": 0, "lighteval|mmlu:high_school_computer_science|5": 0, "lighteval|mmlu:high_school_european_history|5": 0, "lighteval|mmlu:high_school_geography|5": 0, "lighteval|mmlu:high_school_government_and_politics|5": 0, "lighteval|mmlu:high_school_macroeconomics|5": 0, "lighteval|mmlu:high_school_mathematics|5": 0, "lighteval|mmlu:high_school_microeconomics|5": 0, "lighteval|mmlu:high_school_physics|5": 0, "lighteval|mmlu:high_school_psychology|5": 0, "lighteval|mmlu:high_school_statistics|5": 0, "lighteval|mmlu:high_school_us_history|5": 0, "lighteval|mmlu:high_school_world_history|5": 0, "lighteval|mmlu:human_aging|5": 0, "lighteval|mmlu:human_sexuality|5": 0, "lighteval|mmlu:international_law|5": 0, "lighteval|mmlu:jurisprudence|5": 0, "lighteval|mmlu:logical_fallacies|5": 0, "lighteval|mmlu:machine_learning|5": 0, "lighteval|mmlu:management|5": 0, "lighteval|mmlu:marketing|5": 0, "lighteval|mmlu:medical_genetics|5": 0, "lighteval|mmlu:miscellaneous|5": 0, "lighteval|mmlu:moral_disputes|5": 0, "lighteval|mmlu:moral_scenarios|5": 0, "lighteval|mmlu:nutrition|5": 0, "lighteval|mmlu:philosophy|5": 0, "lighteval|mmlu:prehistory|5": 0, "lighteval|mmlu:professional_accounting|5": 0, "lighteval|mmlu:professional_law|5": 0, "lighteval|mmlu:professional_medicine|5": 0, "lighteval|mmlu:professional_psychology|5": 0, "lighteval|mmlu:public_relations|5": 0, "lighteval|mmlu:security_studies|5": 0, "lighteval|mmlu:sociology|5": 0, "lighteval|mmlu:us_foreign_policy|5": 0, "lighteval|mmlu:virology|5": 0, "lighteval|mmlu:world_religions|5": 0 }, "config_tasks": { "lighteval|mmlu:abstract_algebra": { "name": "mmlu:abstract_algebra", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "abstract_algebra", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:anatomy": { "name": "mmlu:anatomy", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "anatomy", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 135, "effective_num_docs": 135 }, "lighteval|mmlu:astronomy": { "name": "mmlu:astronomy", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "astronomy", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 152, "effective_num_docs": 152 }, "lighteval|mmlu:business_ethics": { "name": "mmlu:business_ethics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "business_ethics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:clinical_knowledge": { "name": "mmlu:clinical_knowledge", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "clinical_knowledge", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 265, "effective_num_docs": 265 }, "lighteval|mmlu:college_biology": { "name": "mmlu:college_biology", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "college_biology", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 144, "effective_num_docs": 144 }, "lighteval|mmlu:college_chemistry": { "name": "mmlu:college_chemistry", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "college_chemistry", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:college_computer_science": { "name": "mmlu:college_computer_science", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "college_computer_science", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:college_mathematics": { "name": "mmlu:college_mathematics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "college_mathematics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:college_medicine": { "name": "mmlu:college_medicine", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "college_medicine", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 173, "effective_num_docs": 173 }, "lighteval|mmlu:college_physics": { "name": "mmlu:college_physics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "college_physics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 102, "effective_num_docs": 102 }, "lighteval|mmlu:computer_security": { "name": "mmlu:computer_security", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "computer_security", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:conceptual_physics": { "name": "mmlu:conceptual_physics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "conceptual_physics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 235, "effective_num_docs": 235 }, "lighteval|mmlu:econometrics": { "name": "mmlu:econometrics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "econometrics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 114, "effective_num_docs": 114 }, "lighteval|mmlu:electrical_engineering": { "name": "mmlu:electrical_engineering", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "electrical_engineering", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 145, "effective_num_docs": 145 }, "lighteval|mmlu:elementary_mathematics": { "name": "mmlu:elementary_mathematics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "elementary_mathematics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 378, "effective_num_docs": 378 }, "lighteval|mmlu:formal_logic": { "name": "mmlu:formal_logic", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "formal_logic", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 126, "effective_num_docs": 126 }, "lighteval|mmlu:global_facts": { "name": "mmlu:global_facts", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "global_facts", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:high_school_biology": { "name": "mmlu:high_school_biology", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_biology", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 310, "effective_num_docs": 310 }, "lighteval|mmlu:high_school_chemistry": { "name": "mmlu:high_school_chemistry", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_chemistry", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 203, "effective_num_docs": 203 }, "lighteval|mmlu:high_school_computer_science": { "name": "mmlu:high_school_computer_science", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_computer_science", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:high_school_european_history": { "name": "mmlu:high_school_european_history", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_european_history", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 165, "effective_num_docs": 165 }, "lighteval|mmlu:high_school_geography": { "name": "mmlu:high_school_geography", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_geography", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 198, "effective_num_docs": 198 }, "lighteval|mmlu:high_school_government_and_politics": { "name": "mmlu:high_school_government_and_politics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_government_and_politics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 193, "effective_num_docs": 193 }, "lighteval|mmlu:high_school_macroeconomics": { "name": "mmlu:high_school_macroeconomics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_macroeconomics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 390, "effective_num_docs": 390 }, "lighteval|mmlu:high_school_mathematics": { "name": "mmlu:high_school_mathematics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_mathematics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 270, "effective_num_docs": 270 }, "lighteval|mmlu:high_school_microeconomics": { "name": "mmlu:high_school_microeconomics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_microeconomics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 238, "effective_num_docs": 238 }, "lighteval|mmlu:high_school_physics": { "name": "mmlu:high_school_physics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_physics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 151, "effective_num_docs": 151 }, "lighteval|mmlu:high_school_psychology": { "name": "mmlu:high_school_psychology", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_psychology", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 545, "effective_num_docs": 545 }, "lighteval|mmlu:high_school_statistics": { "name": "mmlu:high_school_statistics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_statistics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 216, "effective_num_docs": 216 }, "lighteval|mmlu:high_school_us_history": { "name": "mmlu:high_school_us_history", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_us_history", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 204, "effective_num_docs": 204 }, "lighteval|mmlu:high_school_world_history": { "name": "mmlu:high_school_world_history", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "high_school_world_history", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 237, "effective_num_docs": 237 }, "lighteval|mmlu:human_aging": { "name": "mmlu:human_aging", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "human_aging", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 223, "effective_num_docs": 223 }, "lighteval|mmlu:human_sexuality": { "name": "mmlu:human_sexuality", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "human_sexuality", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 131, "effective_num_docs": 131 }, "lighteval|mmlu:international_law": { "name": "mmlu:international_law", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "international_law", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 121, "effective_num_docs": 121 }, "lighteval|mmlu:jurisprudence": { "name": "mmlu:jurisprudence", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "jurisprudence", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 108, "effective_num_docs": 108 }, "lighteval|mmlu:logical_fallacies": { "name": "mmlu:logical_fallacies", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "logical_fallacies", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 163, "effective_num_docs": 163 }, "lighteval|mmlu:machine_learning": { "name": "mmlu:machine_learning", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "machine_learning", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 112, "effective_num_docs": 112 }, "lighteval|mmlu:management": { "name": "mmlu:management", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "management", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 103, "effective_num_docs": 103 }, "lighteval|mmlu:marketing": { "name": "mmlu:marketing", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "marketing", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 234, "effective_num_docs": 234 }, "lighteval|mmlu:medical_genetics": { "name": "mmlu:medical_genetics", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "medical_genetics", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:miscellaneous": { "name": "mmlu:miscellaneous", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "miscellaneous", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 783, "effective_num_docs": 783 }, "lighteval|mmlu:moral_disputes": { "name": "mmlu:moral_disputes", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "moral_disputes", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 346, "effective_num_docs": 346 }, "lighteval|mmlu:moral_scenarios": { "name": "mmlu:moral_scenarios", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "moral_scenarios", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 895, "effective_num_docs": 895 }, "lighteval|mmlu:nutrition": { "name": "mmlu:nutrition", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "nutrition", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 306, "effective_num_docs": 306 }, "lighteval|mmlu:philosophy": { "name": "mmlu:philosophy", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "philosophy", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 311, "effective_num_docs": 311 }, "lighteval|mmlu:prehistory": { "name": "mmlu:prehistory", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "prehistory", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 324, "effective_num_docs": 324 }, "lighteval|mmlu:professional_accounting": { "name": "mmlu:professional_accounting", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "professional_accounting", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 282, "effective_num_docs": 282 }, "lighteval|mmlu:professional_law": { "name": "mmlu:professional_law", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "professional_law", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 1534, "effective_num_docs": 1534 }, "lighteval|mmlu:professional_medicine": { "name": "mmlu:professional_medicine", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "professional_medicine", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 272, "effective_num_docs": 272 }, "lighteval|mmlu:professional_psychology": { "name": "mmlu:professional_psychology", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "professional_psychology", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 612, "effective_num_docs": 612 }, "lighteval|mmlu:public_relations": { "name": "mmlu:public_relations", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "public_relations", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 110, "effective_num_docs": 110 }, "lighteval|mmlu:security_studies": { "name": "mmlu:security_studies", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "security_studies", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 245, "effective_num_docs": 245 }, "lighteval|mmlu:sociology": { "name": "mmlu:sociology", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "sociology", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 201, "effective_num_docs": 201 }, "lighteval|mmlu:us_foreign_policy": { "name": "mmlu:us_foreign_policy", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "us_foreign_policy", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 100, "effective_num_docs": 100 }, "lighteval|mmlu:virology": { "name": "mmlu:virology", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "virology", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 166, "effective_num_docs": 166 }, "lighteval|mmlu:world_religions": { "name": "mmlu:world_religions", "prompt_function": "mmlu_harness", "hf_repo": "lighteval/mmlu", "hf_subset": "world_religions", "metric": [ "loglikelihood_acc" ], "hf_avail_splits": [ "auxiliary_train", "test", "validation", "dev" ], "evaluation_splits": [ "test" ], "few_shots_split": "dev", "few_shots_select": "sequential", "generation_size": 1, "stop_sequence": [ "\n" ], "output_regex": null, "frozen": false, "suite": [ "lighteval", "mmlu" ], "original_num_docs": 171, "effective_num_docs": 171 } }, "summary_tasks": { "lighteval|mmlu:abstract_algebra|5": { "hashes": { "hash_examples": "4c76229e00c9c0e9", "hash_full_prompts": "a45d01c3409c889c", "hash_input_tokens": "6061ef74e94498a1", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:anatomy|5": { "hashes": { "hash_examples": "6a1f8104dccbd33b", "hash_full_prompts": "e245c6600e03cc32", "hash_input_tokens": "1906e583b9621b5c", "hash_cont_tokens": "025910e68cf29c3d" }, "truncated": 0, "non_truncated": 135, "padded": 540, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:astronomy|5": { "hashes": { "hash_examples": "1302effa3a76ce4c", "hash_full_prompts": "390f9bddf857ad04", "hash_input_tokens": "771fefaee108888d", "hash_cont_tokens": "1a66fd04f03e0517" }, "truncated": 0, "non_truncated": 152, "padded": 608, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:business_ethics|5": { "hashes": { "hash_examples": "03cb8bce5336419a", "hash_full_prompts": "5504f893bc4f2fa1", "hash_input_tokens": "d94fae0608621dd6", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:clinical_knowledge|5": { "hashes": { "hash_examples": "ffbb9c7b2be257f9", "hash_full_prompts": "106ad0bab4b90b78", "hash_input_tokens": "e8436d40a9c05ea3", "hash_cont_tokens": "de872053260a1588" }, "truncated": 0, "non_truncated": 265, "padded": 1060, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:college_biology|5": { "hashes": { "hash_examples": "3ee77f176f38eb8e", "hash_full_prompts": "59f9bdf2695cb226", "hash_input_tokens": "1bc1db858463d32a", "hash_cont_tokens": "9ace296b3e00bba3" }, "truncated": 0, "non_truncated": 144, "padded": 576, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:college_chemistry|5": { "hashes": { "hash_examples": "ce61a69c46d47aeb", "hash_full_prompts": "3cac9b759fcff7a0", "hash_input_tokens": "2a3c52842d61ce67", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:college_computer_science|5": { "hashes": { "hash_examples": "32805b52d7d5daab", "hash_full_prompts": "010b0cca35070130", "hash_input_tokens": "a6a78a8854b3ff3c", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:college_mathematics|5": { "hashes": { "hash_examples": "55da1a0a0bd33722", "hash_full_prompts": "511422eb9eefc773", "hash_input_tokens": "9067af15a43e3943", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:college_medicine|5": { "hashes": { "hash_examples": "c33e143163049176", "hash_full_prompts": "c8cc1a82a51a046e", "hash_input_tokens": "d4973e45d254d606", "hash_cont_tokens": "c80c0b5489bdbc5a" }, "truncated": 0, "non_truncated": 173, "padded": 692, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:college_physics|5": { "hashes": { "hash_examples": "ebdab1cdb7e555df", "hash_full_prompts": "e40721b5059c5818", "hash_input_tokens": "974434dd212cc64c", "hash_cont_tokens": "569fcb9ac44734ae" }, "truncated": 0, "non_truncated": 102, "padded": 408, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:computer_security|5": { "hashes": { "hash_examples": "a24fd7d08a560921", "hash_full_prompts": "946c9be5964ac44a", "hash_input_tokens": "03a6468f366cc5c9", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:conceptual_physics|5": { "hashes": { "hash_examples": "8300977a79386993", "hash_full_prompts": "506a4f6094cc40c9", "hash_input_tokens": "f534b68e21be3b1a", "hash_cont_tokens": "6e88c64c1a76752a" }, "truncated": 0, "non_truncated": 235, "padded": 940, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:econometrics|5": { "hashes": { "hash_examples": "ddde36788a04a46f", "hash_full_prompts": "4ed2703f27f1ed05", "hash_input_tokens": "8ff52a6059f5bdd9", "hash_cont_tokens": "a315e0e16c922c3c" }, "truncated": 0, "non_truncated": 114, "padded": 456, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:electrical_engineering|5": { "hashes": { "hash_examples": "acbc5def98c19b3f", "hash_full_prompts": "d8f4b3e11c23653c", "hash_input_tokens": "c33c375027d89492", "hash_cont_tokens": "44c72e6a7422c304" }, "truncated": 0, "non_truncated": 145, "padded": 580, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:elementary_mathematics|5": { "hashes": { "hash_examples": "146e61d07497a9bd", "hash_full_prompts": "256d111bd15647ff", "hash_input_tokens": "2ef5113cea686f48", "hash_cont_tokens": "cac0a6c304791bb7" }, "truncated": 0, "non_truncated": 378, "padded": 1512, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:formal_logic|5": { "hashes": { "hash_examples": "8635216e1909a03f", "hash_full_prompts": "1171d04f3b1a11f5", "hash_input_tokens": "ec97e1c379123fff", "hash_cont_tokens": "8801fad3bbc72e57" }, "truncated": 0, "non_truncated": 126, "padded": 504, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:global_facts|5": { "hashes": { "hash_examples": "30b315aa6353ee47", "hash_full_prompts": "a7e56dbc074c7529", "hash_input_tokens": "2b0807209002d279", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_biology|5": { "hashes": { "hash_examples": "c9136373af2180de", "hash_full_prompts": "ad6e859ed978e04a", "hash_input_tokens": "e8cadacc947daa13", "hash_cont_tokens": "2d57d9e2c5a1fd64" }, "truncated": 0, "non_truncated": 310, "padded": 1240, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_chemistry|5": { "hashes": { "hash_examples": "b0661bfa1add6404", "hash_full_prompts": "6eb9c04bcc8a8f2a", "hash_input_tokens": "8ae6b3d5212ac67f", "hash_cont_tokens": "bb0fd92673ddfb31" }, "truncated": 0, "non_truncated": 203, "padded": 812, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_computer_science|5": { "hashes": { "hash_examples": "80fc1d623a3d665f", "hash_full_prompts": "8e51bc91c81cf8dd", "hash_input_tokens": "46dfdc239de27d6e", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_european_history|5": { "hashes": { "hash_examples": "854da6e5af0fe1a1", "hash_full_prompts": "664a1f16c9f3195c", "hash_input_tokens": "a3c136d3ff165617", "hash_cont_tokens": "16e494cddccc4a04" }, "truncated": 0, "non_truncated": 165, "padded": 656, "non_padded": 4, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_geography|5": { "hashes": { "hash_examples": "7dc963c7acd19ad8", "hash_full_prompts": "f3acf911f4023c8a", "hash_input_tokens": "e1360551f2ad3e47", "hash_cont_tokens": "16b7f65a07b3d47b" }, "truncated": 0, "non_truncated": 198, "padded": 792, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_government_and_politics|5": { "hashes": { "hash_examples": "1f675dcdebc9758f", "hash_full_prompts": "066254feaa3158ae", "hash_input_tokens": "6564364e1f4d4c79", "hash_cont_tokens": "476e87fd675136aa" }, "truncated": 0, "non_truncated": 193, "padded": 772, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_macroeconomics|5": { "hashes": { "hash_examples": "2fb32cf2d80f0b35", "hash_full_prompts": "19a7fa502aa85c95", "hash_input_tokens": "d59bdfdf903bf379", "hash_cont_tokens": "b0c7b4c5f7bdf3e7" }, "truncated": 0, "non_truncated": 390, "padded": 1560, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_mathematics|5": { "hashes": { "hash_examples": "fd6646fdb5d58a1f", "hash_full_prompts": "4f704e369778b5b0", "hash_input_tokens": "b3a5c15701024f1f", "hash_cont_tokens": "1a05d6ff49846fd1" }, "truncated": 0, "non_truncated": 270, "padded": 1080, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_microeconomics|5": { "hashes": { "hash_examples": "2118f21f71d87d84", "hash_full_prompts": "4350f9e2240f8010", "hash_input_tokens": "69e3156cb9bbbda9", "hash_cont_tokens": "0e7f0645ffffd6cd" }, "truncated": 0, "non_truncated": 238, "padded": 949, "non_padded": 3, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_physics|5": { "hashes": { "hash_examples": "dc3ce06378548565", "hash_full_prompts": "5dc0d6831b66188f", "hash_input_tokens": "ba950d9d4f5d5e47", "hash_cont_tokens": "41ca6560b8c10183" }, "truncated": 0, "non_truncated": 151, "padded": 604, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_psychology|5": { "hashes": { "hash_examples": "c8d1d98a40e11f2f", "hash_full_prompts": "af2b097da6d50365", "hash_input_tokens": "a54adefaeb5500a2", "hash_cont_tokens": "53a17ff85c607844" }, "truncated": 0, "non_truncated": 545, "padded": 2178, "non_padded": 2, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_statistics|5": { "hashes": { "hash_examples": "666c8759b98ee4ff", "hash_full_prompts": "c757694421d6d68d", "hash_input_tokens": "240419bd9137ca86", "hash_cont_tokens": "bc9063ad140cc941" }, "truncated": 0, "non_truncated": 216, "padded": 864, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_us_history|5": { "hashes": { "hash_examples": "95fef1c4b7d3f81e", "hash_full_prompts": "e34a028d0ddeec5e", "hash_input_tokens": "b27e2d090c7722d6", "hash_cont_tokens": "5cf777085ba01096" }, "truncated": 0, "non_truncated": 204, "padded": 816, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:high_school_world_history|5": { "hashes": { "hash_examples": "7e5085b6184b0322", "hash_full_prompts": "1fa3d51392765601", "hash_input_tokens": "a0503454c72d62f4", "hash_cont_tokens": "152af2d9e4830517" }, "truncated": 0, "non_truncated": 237, "padded": 948, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:human_aging|5": { "hashes": { "hash_examples": "c17333e7c7c10797", "hash_full_prompts": "cac900721f9a1a94", "hash_input_tokens": "cb1a5f4e8d8ece53", "hash_cont_tokens": "da4d9eaa044021dd" }, "truncated": 0, "non_truncated": 223, "padded": 892, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:human_sexuality|5": { "hashes": { "hash_examples": "4edd1e9045df5e3d", "hash_full_prompts": "0d6567bafee0a13c", "hash_input_tokens": "06f0a1b69fc0f6e9", "hash_cont_tokens": "1b99e384258a4eeb" }, "truncated": 0, "non_truncated": 131, "padded": 524, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:international_law|5": { "hashes": { "hash_examples": "db2fa00d771a062a", "hash_full_prompts": "d018f9116479795e", "hash_input_tokens": "24cda8ff066603d5", "hash_cont_tokens": "cbf02c30cdded208" }, "truncated": 0, "non_truncated": 121, "padded": 484, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:jurisprudence|5": { "hashes": { "hash_examples": "e956f86b124076fe", "hash_full_prompts": "1487e89a10ec58b7", "hash_input_tokens": "6a574eda8bdea629", "hash_cont_tokens": "4b248cf879d97a50" }, "truncated": 0, "non_truncated": 108, "padded": 424, "non_padded": 8, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:logical_fallacies|5": { "hashes": { "hash_examples": "956e0e6365ab79f1", "hash_full_prompts": "677785b2181f9243", "hash_input_tokens": "79da057bb404ffdd", "hash_cont_tokens": "6d9c35172b158838" }, "truncated": 0, "non_truncated": 163, "padded": 632, "non_padded": 20, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:machine_learning|5": { "hashes": { "hash_examples": "397997cc6f4d581e", "hash_full_prompts": "769ee14a2aea49bb", "hash_input_tokens": "1368f69e651d7a49", "hash_cont_tokens": "66c3ec85fee2fc98" }, "truncated": 0, "non_truncated": 112, "padded": 448, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:management|5": { "hashes": { "hash_examples": "2bcbe6f6ca63d740", "hash_full_prompts": "cb1ff9dac9582144", "hash_input_tokens": "4932314520ff5a1c", "hash_cont_tokens": "5e2470abd1fb9d10" }, "truncated": 0, "non_truncated": 103, "padded": 412, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:marketing|5": { "hashes": { "hash_examples": "8ddb20d964a1b065", "hash_full_prompts": "9fc2114a187ad9a2", "hash_input_tokens": "c566f215df327b23", "hash_cont_tokens": "27fe68d9630f8999" }, "truncated": 0, "non_truncated": 234, "padded": 916, "non_padded": 20, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:medical_genetics|5": { "hashes": { "hash_examples": "182a71f4763d2cea", "hash_full_prompts": "46a616fa51878959", "hash_input_tokens": "2d52eb85effca1ce", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:miscellaneous|5": { "hashes": { "hash_examples": "4c404fdbb4ca57fc", "hash_full_prompts": "0813e1be36dbaae1", "hash_input_tokens": "8ab40049dc296984", "hash_cont_tokens": "dfa423a160edd337" }, "truncated": 0, "non_truncated": 783, "padded": 3128, "non_padded": 4, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:moral_disputes|5": { "hashes": { "hash_examples": "60cbd2baa3fea5c9", "hash_full_prompts": "1d14adebb9b62519", "hash_input_tokens": "4c5f82d25a8ebf96", "hash_cont_tokens": "bef966e6669349be" }, "truncated": 0, "non_truncated": 346, "padded": 1380, "non_padded": 4, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:moral_scenarios|5": { "hashes": { "hash_examples": "fd8b0431fbdd75ef", "hash_full_prompts": "b80d3d236165e3de", "hash_input_tokens": "72d00d71f8182e56", "hash_cont_tokens": "a7bfdd944d86bcb5" }, "truncated": 0, "non_truncated": 895, "padded": 3575, "non_padded": 5, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:nutrition|5": { "hashes": { "hash_examples": "71e55e2b829b6528", "hash_full_prompts": "2bfb18e5fab8dea7", "hash_input_tokens": "1c01b894ae13a2f4", "hash_cont_tokens": "fcda7736026f2449" }, "truncated": 0, "non_truncated": 306, "padded": 1224, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:philosophy|5": { "hashes": { "hash_examples": "a6d489a8d208fa4b", "hash_full_prompts": "e8c0d5b6dae3ccc8", "hash_input_tokens": "344b97bbc55eadeb", "hash_cont_tokens": "0f39b851342e8986" }, "truncated": 0, "non_truncated": 311, "padded": 1244, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:prehistory|5": { "hashes": { "hash_examples": "6cc50f032a19acaa", "hash_full_prompts": "4a6a1d3ab1bf28e4", "hash_input_tokens": "e2270c2e739d565b", "hash_cont_tokens": "b60e45d3e9856b35" }, "truncated": 0, "non_truncated": 324, "padded": 1280, "non_padded": 16, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:professional_accounting|5": { "hashes": { "hash_examples": "50f57ab32f5f6cea", "hash_full_prompts": "e60129bd2d82ffc6", "hash_input_tokens": "b6e8898d96ed525d", "hash_cont_tokens": "a0c4e121b7293818" }, "truncated": 0, "non_truncated": 282, "padded": 1112, "non_padded": 16, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:professional_law|5": { "hashes": { "hash_examples": "a8fdc85c64f4b215", "hash_full_prompts": "0dbb1d9b72dcea03", "hash_input_tokens": "71053a5cb26234dc", "hash_cont_tokens": "68b662abeba54fbc" }, "truncated": 0, "non_truncated": 1534, "padded": 6136, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:professional_medicine|5": { "hashes": { "hash_examples": "c373a28a3050a73a", "hash_full_prompts": "5e040f9ca68b089e", "hash_input_tokens": "b43ad52177e11456", "hash_cont_tokens": "6caeac5412bb4a09" }, "truncated": 0, "non_truncated": 272, "padded": 1088, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:professional_psychology|5": { "hashes": { "hash_examples": "bf5254fe818356af", "hash_full_prompts": "b386ecda8b87150e", "hash_input_tokens": "c250d3866bd3c8d8", "hash_cont_tokens": "79b091252a1095a9" }, "truncated": 0, "non_truncated": 612, "padded": 2448, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:public_relations|5": { "hashes": { "hash_examples": "b66d52e28e7d14e0", "hash_full_prompts": "fe43562263e25677", "hash_input_tokens": "d4b9f802b45946f8", "hash_cont_tokens": "987115a77c8704f0" }, "truncated": 0, "non_truncated": 110, "padded": 436, "non_padded": 4, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:security_studies|5": { "hashes": { "hash_examples": "514c14feaf000ad9", "hash_full_prompts": "27d4a2ac541ef4b9", "hash_input_tokens": "dd7c51a906f3bf73", "hash_cont_tokens": "6c35bc7e96074b27" }, "truncated": 0, "non_truncated": 245, "padded": 980, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:sociology|5": { "hashes": { "hash_examples": "f6c9bc9d18c80870", "hash_full_prompts": "c072ea7d1a1524f2", "hash_input_tokens": "50d0b11b96edc318", "hash_cont_tokens": "32af622f73b2e657" }, "truncated": 0, "non_truncated": 201, "padded": 804, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:us_foreign_policy|5": { "hashes": { "hash_examples": "ed7b78629db6678f", "hash_full_prompts": "341a97ca3e4d699d", "hash_input_tokens": "f16b53370750082e", "hash_cont_tokens": "9e1c9ca2c51de57e" }, "truncated": 0, "non_truncated": 100, "padded": 400, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:virology|5": { "hashes": { "hash_examples": "bc52ffdc3f9b994a", "hash_full_prompts": "651d471e2eb8b5e9", "hash_input_tokens": "61ad3b476830b6f3", "hash_cont_tokens": "beded8c3660dc8f5" }, "truncated": 0, "non_truncated": 166, "padded": 664, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 }, "lighteval|mmlu:world_religions|5": { "hashes": { "hash_examples": "ecdb4a4f94f62930", "hash_full_prompts": "3773f03542ce44a3", "hash_input_tokens": "804fecc55001e28a", "hash_cont_tokens": "9b1952a4af3d6a73" }, "truncated": 0, "non_truncated": 171, "padded": 684, "non_padded": 0, "effective_few_shots": 5.0, "num_truncated_few_shots": 0 } }, "summary_general": { "hashes": { "hash_examples": "341a076d0beb7048", "hash_full_prompts": "a5c8f2b7ff4f5ae2", "hash_input_tokens": "9273ebae51e8421d", "hash_cont_tokens": "25e9f343d6b95644" }, "truncated": 0, "non_truncated": 14042, "padded": 56062, "non_padded": 106, "num_truncated_few_shots": 0 } }