whisperkit-benchmarks / dashboard_data /test_coverage_112a023.json
ardaatahan's picture
Add parity, test coverage and distinguish between device support and test coverage
d3a1d6c
raw
history blame
4.01 kB
{
"commit_hash": "112a023",
"total_devices": 137,
"tested_devices": 49,
"skipped_devices": 88,
"coverage_percentage": 35.76642335766424,
"tested_device_list": [
"iPhone12,3",
"Mac15,10",
"iPad14,9",
"iPhone12,1",
"Mac14,15",
"iPhone14,6",
"Mac15,13",
"Mac14,6",
"iPhone14,3",
"Mac14,13",
"Mac14,5",
"iPhone14,8",
"iPhone14,5",
"iPhone14,2",
"iPad14,6",
"Mac14,12",
"Mac16,9",
"Mac14,8",
"Mac15,12",
"Mac14,2",
"iPad14,10",
"iPhone14,7",
"Mac14,10",
"iPad16,6",
"iPad16,3",
"iPad16,5",
"Mac15,5",
"iPhone14,4",
"Mac14,14",
"Mac14,7",
"Mac15,11",
"iPad14,3",
"iPad14,5",
"Mac14,9",
"Mac15,6",
"iPad16,4",
"Mac15,4",
"Mac15,8",
"Mac16,8",
"Mac14,4",
"Mac15,9",
"Mac15,3",
"iPad14,4",
"Mac14,3",
"iPhone12,8",
"iPad14,11",
"Mac15,7",
"iPhone12,5",
"iPad14,8"
],
"skipped_device_list": [
"iPad15,5",
"iPad13,2",
"iPad8,6",
"iPad8,12",
"iPad8,9",
"iPad15,7",
"iPad15,3",
"iPad13,8",
"Mac16,6",
"iPhone11,5",
"iPad13,19",
"iPad8,2",
"iPhone16,1",
"iPhone12,2",
"iPad13,7",
"iPad8,3",
"iPhone13,1",
"iPad16,2",
"Macmini9,1",
"MacBookPro18,2",
"iPhone11,8",
"MacBookPro17,1",
"Mac16,10",
"iPhone15,5",
"iPhone17,4",
"Mac13,1",
"MacBookAir10,1",
"Mac13,2",
"iPad13,10",
"iPad8,11",
"Mac16,13",
"iPad13,1",
"iPad8,10",
"iPad13,5",
"iPad14,2",
"iPad13,9",
"iPhone15,2",
"iPad11,3",
"iPhone13,2",
"iPhone17,5",
"MacBookPro18,1",
"iMac21,2",
"MacBookPro18,4",
"iPad13,6",
"iPhone11,4",
"iPad8,1",
"iPhone17,2",
"iPhone11,6",
"iPad12,1",
"iMac21,1",
"MacBookPro18,3",
"iPhone11,3",
"iPad8,4",
"iPad11,4",
"iPad13,18",
"Mac16,3",
"iPhone12,4",
"iPhone13,3",
"Mac16,5",
"iPhone17,3",
"iPad8,7",
"iPad13,4",
"iPad15,8",
"iPad11,1",
"iPad11,2",
"iPhone13,4",
"iPad14,1",
"iPad13,17",
"iPad16,1",
"iPhone16,2",
"iPad15,4",
"iPhone11,2",
"iPhone15,4",
"iPhone17,1",
"iPad13,11",
"iPad8,5",
"Mac16,1",
"Mac16,2",
"iPad11,7",
"iPad11,6",
"Mac16,7",
"iPad13,16",
"iPhone15,3",
"iPad8,8",
"Mac16,12",
"iPad15,6",
"Mac16,11",
"iPad12,2"
],
"tested_os_versions": [
"macOS_15.1.1",
"iPadOS_18.1",
"iPadOS_17.6.1",
"macOS_15.0.1",
"macOS_15.2.0",
"iOS_17.6.1",
"iOS_17.7.1",
"macOS_15.2"
],
"has_target_os_coverage": false,
"covered_target_versions": [
"macOS 15",
"iOS 17",
"iOS 18"
],
"missing_target_versions": [
"macOS 14",
"macOS 26",
"iOS 26"
],
"has_target_chip_coverage": false,
"platform_chip_coverage": {
"iPad": {
"total_chips": 8,
"tested_chips": 2,
"coverage_percentage": 25.0,
"covered_chips": [
"M2",
"M4"
],
"missing_chips": [
"A14",
"A15",
"A16",
"A17 Pro",
"M1",
"M3"
]
},
"iPhone": {
"total_chips": 6,
"tested_chips": 2,
"coverage_percentage": 33.33333333333333,
"covered_chips": [
"A13",
"A15"
],
"missing_chips": [
"A14",
"A16",
"A17 Pro",
"A18",
"A18 Pro"
]
},
"Mac": {
"total_chips": 4,
"tested_chips": 2,
"coverage_percentage": 50.0,
"covered_chips": [
"M2",
"M3"
],
"missing_chips": [
"M1",
"M4"
]
}
},
"missing_target_chips": {
"iPad": [
"A14",
"A15",
"A16",
"A17 Pro",
"M1",
"M3"
],
"iPhone": [
"A14",
"A16",
"A17 Pro",
"A18",
"A18 Pro"
],
"Mac": [
"M1",
"M4"
]
}
}