edbeeching's picture
edbeeching HF staff
Upload eval_results/HuggingFaceH4/mistral-7b-ift/v41.25/ifeval/results_2024-03-13T08-55-56.624807.json with huggingface_hub
9f90b40 verified
raw
history blame
2.47 kB
{
"config_general": {
"lighteval_sha": "?",
"num_fewshot_seeds": 1,
"override_batch_size": 4,
"max_samples": null,
"job_id": "",
"start_time": 196007.215023199,
"end_time": 196827.101976067,
"total_evaluation_time_secondes": "819.8869528679934",
"model_name": "HuggingFaceH4/mistral-7b-ift",
"model_sha": "a0c310fe5a2fd163f48f1f3fdd7e9cde20c50e11",
"model_dtype": "torch.bfloat16",
"model_size": "13.99 GB",
"config": null
},
"results": {
"custom|ifeval|0": {
"prompt_level_strict_acc": 0.35304990757855825,
"prompt_level_strict_acc_stderr": 0.020566318668824662,
"inst_level_strict_acc": 0.46642685851318944,
"inst_level_strict_acc_stderr": 0.0005445081650805074,
"prompt_level_loose_acc": 0.3826247689463956,
"prompt_level_loose_acc_stderr": 0.02091530784143665,
"inst_level_loose_acc": 0.49760191846522783,
"inst_level_loose_acc_stderr": 0.0005523084336440809
}
},
"versions": {
"custom|ifeval|0": 0
},
"config_tasks": {
"custom|ifeval": {
"name": "ifeval",
"prompt_function": "ifeval_prompt",
"hf_repo": "wis-k/instruction-following-eval",
"hf_subset": "default",
"metric": [
"ifeval_metric"
],
"hf_avail_splits": [
"train"
],
"evaluation_splits": [
"train"
],
"few_shots_split": "train",
"few_shots_select": "random_sampling",
"generation_size": 1280,
"stop_sequence": [],
"output_regex": null,
"frozen": false,
"suite": [
"custom"
],
"original_num_docs": 541,
"effective_num_docs": 541
}
},
"summary_tasks": {
"custom|ifeval|0": {
"hashes": {
"hash_examples": "e99cbf567588d7c6",
"hash_full_prompts": "46932ca113ba6b0c",
"hash_input_tokens": "8b2e0300fbfb70bc",
"hash_cont_tokens": "df345f0bb8ca0cef"
},
"truncated": 0,
"non_truncated": 541,
"padded": 0,
"non_padded": 541,
"effective_few_shots": 0.0,
"num_truncated_few_shots": 0
}
},
"summary_general": {
"hashes": {
"hash_examples": "ea046ab2c6fc5928",
"hash_full_prompts": "8ed52f7b74eb0590",
"hash_input_tokens": "9f4b7e9e7ff95d4a",
"hash_cont_tokens": "e77b1d15f9ed980a"
},
"truncated": 0,
"non_truncated": 541,
"padded": 0,
"non_padded": 541,
"num_truncated_few_shots": 0
}
}