File size: 2,505 Bytes
c8fc538 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
{
"config_general": {
"lighteval_sha": "?",
"num_fewshot_seeds": 1,
"override_batch_size": 4,
"max_samples": null,
"job_id": "",
"start_time": 133507.195380292,
"end_time": 134078.325022514,
"total_evaluation_time_secondes": "571.1296422220184",
"model_name": "HuggingFaceH4/qwen-1.5-0.5b-ift",
"model_sha": "6206c32a21e44c32627395128f39aee77d3739a6",
"model_dtype": "torch.bfloat16",
"model_size": "1.05 GB",
"config": null
},
"results": {
"custom|ifeval|0": {
"prompt_level_strict_acc": 0.1756007393715342,
"prompt_level_strict_acc_stderr": 0.01637325731205791,
"inst_level_strict_acc": 0.30335731414868106,
"inst_level_strict_acc_stderr": 0.0004804332882065046,
"prompt_level_loose_acc": 0.18299445471349354,
"prompt_level_loose_acc_stderr": 0.016639282183680784,
"inst_level_loose_acc": 0.31894484412470026,
"inst_level_loose_acc_stderr": 0.0004909221842397014
}
},
"versions": {
"custom|ifeval|0": 0
},
"config_tasks": {
"custom|ifeval": {
"name": "ifeval",
"prompt_function": "ifeval_prompt",
"hf_repo": "wis-k/instruction-following-eval",
"hf_subset": "default",
"metric": [
"ifeval_metric"
],
"hf_avail_splits": [
"train"
],
"evaluation_splits": [
"train"
],
"few_shots_split": "train",
"few_shots_select": "random_sampling",
"generation_size": 1280,
"stop_sequence": [],
"output_regex": null,
"frozen": false,
"suite": [
"custom"
],
"original_num_docs": 541,
"effective_num_docs": 541,
"trust_dataset": null
}
},
"summary_tasks": {
"custom|ifeval|0": {
"hashes": {
"hash_examples": "e99cbf567588d7c6",
"hash_full_prompts": "7ea7bf2a8edba8f4",
"hash_input_tokens": "406f216e2e739d61",
"hash_cont_tokens": "dd4b50d160a7b4a9"
},
"truncated": 0,
"non_truncated": 541,
"padded": 0,
"non_padded": 541,
"effective_few_shots": 0.0,
"num_truncated_few_shots": 0
}
},
"summary_general": {
"hashes": {
"hash_examples": "ea046ab2c6fc5928",
"hash_full_prompts": "45f8422f6ad2da79",
"hash_input_tokens": "7be1d930c9579e40",
"hash_cont_tokens": "82547e50b3be0144"
},
"truncated": 0,
"non_truncated": 541,
"padded": 0,
"non_padded": 541,
"num_truncated_few_shots": 0
}
} |