{ "syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.97, "ndcg_at_3": 0.98893, "ndcg_at_5": 0.98893, "ndcg_at_10": 0.98893, "ndcg_at_20": 0.98893, "ndcg_at_50": 0.98893, "ndcg_at_100": 0.98893, "map_at_1": 0.97, "map_at_3": 0.985, "map_at_5": 0.985, "map_at_10": 0.985, "map_at_20": 0.985, "map_at_50": 0.985, "map_at_100": 0.985, "recall_at_1": 0.97, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.98, "mrr_at_3": 0.99, "mrr_at_5": 0.99, "mrr_at_10": 0.99, "mrr_at_20": 0.99, "mrr_at_50": 0.99, "mrr_at_100": 0.99, "naucs_at_1_max": 0.24696545284780627, "naucs_at_1_std": 0.24696545284780627, "naucs_at_1_diff1": 1.0, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "tabfquad_test_subsampled": { "ndcg_at_1": 0.88571, "ndcg_at_3": 0.93567, "ndcg_at_5": 0.93875, "ndcg_at_10": 0.94242, "ndcg_at_20": 0.94338, "ndcg_at_50": 0.94556, "ndcg_at_100": 0.94556, "map_at_1": 0.88571, "map_at_3": 0.9244, "map_at_5": 0.92619, "map_at_10": 0.92783, "map_at_20": 0.92812, "map_at_50": 0.92849, "map_at_100": 0.92849, "recall_at_1": 0.88571, "recall_at_3": 0.96786, "recall_at_5": 0.975, "recall_at_10": 0.98571, "recall_at_20": 0.98929, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.88571, "precision_at_3": 0.32262, "precision_at_5": 0.195, "precision_at_10": 0.09857, "precision_at_20": 0.04946, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8857142857142857, "mrr_at_3": 0.9255952380952379, "mrr_at_5": 0.9280952380952381, "mrr_at_10": 0.9291369047619047, "mrr_at_20": 0.9294345238095238, "mrr_at_50": 0.9298080357142857, "mrr_at_100": 0.9298080357142857, "naucs_at_1_max": 0.6126156858092077, "naucs_at_1_std": 0.33993830090175703, "naucs_at_1_diff1": 0.9092311343141914, "naucs_at_3_max": 0.919597468617075, "naucs_at_3_std": 0.8003423591658857, "naucs_at_3_diff1": 0.9854756717501797, "naucs_at_5_max": 0.9603174603174579, "naucs_at_5_std": 0.9603174603174579, "naucs_at_5_diff1": 0.981325863678801, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "shiftproject_test": { "ndcg_at_1": 0.79, "ndcg_at_3": 0.88309, "ndcg_at_5": 0.89127, "ndcg_at_10": 0.89416, "ndcg_at_20": 0.89416, "ndcg_at_50": 0.89828, "ndcg_at_100": 0.89828, "map_at_1": 0.79, "map_at_3": 0.86, "map_at_5": 0.8645, "map_at_10": 0.8655, "map_at_20": 0.8655, "map_at_50": 0.86622, "map_at_100": 0.86622, "recall_at_1": 0.79, "recall_at_3": 0.95, "recall_at_5": 0.97, "recall_at_10": 0.98, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.79, "precision_at_3": 0.31667, "precision_at_5": 0.194, "precision_at_10": 0.098, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8, "mrr_at_3": 0.8683333333333333, "mrr_at_5": 0.8728333333333333, "mrr_at_10": 0.8728333333333333, "mrr_at_20": 0.8737424242424242, "mrr_at_50": 0.8745417972831766, "mrr_at_100": 0.8745417972831766, "naucs_at_1_max": 0.057974847909202674, "naucs_at_1_std": -0.42724889989660236, "naucs_at_1_diff1": 0.8040493423425586, "naucs_at_3_max": 0.09589169000933911, "naucs_at_3_std": -0.5948646125116669, "naucs_at_3_diff1": 0.9477124183006521, "naucs_at_5_max": -0.0004668534080290478, "naucs_at_5_std": -0.35434173669467856, "naucs_at_5_diff1": 0.9564270152505424, "naucs_at_10_max": -0.43534080298785716, "naucs_at_10_std": -0.5929038281979383, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": -0.43534080298785716, "naucs_at_20_std": -0.5929038281979383, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.96178, "ndcg_at_5": 0.96178, "ndcg_at_10": 0.96178, "ndcg_at_20": 0.96178, "ndcg_at_50": 0.96178, "ndcg_at_100": 0.96178, "map_at_1": 0.9, "map_at_3": 0.94833, "map_at_5": 0.94833, "map_at_10": 0.94833, "map_at_20": 0.94833, "map_at_50": 0.94833, "map_at_100": 0.94833, "recall_at_1": 0.9, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.9683333333333333, "mrr_at_5": 0.9683333333333333, "mrr_at_10": 0.9683333333333333, "mrr_at_20": 0.9683333333333333, "mrr_at_50": 0.9683333333333333, "mrr_at_100": 0.9683333333333333, "naucs_at_1_max": 0.709710550887022, "naucs_at_1_std": 0.3939775910364137, "naucs_at_1_diff1": 0.9738562091503267, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "infovqa_test_subsampled": { "ndcg_at_1": 0.89271, "ndcg_at_3": 0.9279, "ndcg_at_5": 0.93548, "ndcg_at_10": 0.93812, "ndcg_at_20": 0.93968, "ndcg_at_50": 0.94205, "ndcg_at_100": 0.94241, "map_at_1": 0.89271, "map_at_3": 0.9197, "map_at_5": 0.92395, "map_at_10": 0.92506, "map_at_20": 0.9255, "map_at_50": 0.92586, "map_at_100": 0.9259, "recall_at_1": 0.89271, "recall_at_3": 0.95142, "recall_at_5": 0.96964, "recall_at_10": 0.97773, "recall_at_20": 0.98381, "recall_at_50": 0.99595, "recall_at_100": 0.99798, "precision_at_1": 0.89271, "precision_at_3": 0.31714, "precision_at_5": 0.19393, "precision_at_10": 0.09777, "precision_at_20": 0.04919, "precision_at_50": 0.01992, "precision_at_100": 0.00998, "mrr_at_1": 0.8967611336032388, "mrr_at_3": 0.9220647773279351, "mrr_at_5": 0.9258097165991901, "mrr_at_10": 0.9269503887924939, "mrr_at_20": 0.9273423944476574, "mrr_at_50": 0.9277128849497269, "mrr_at_100": 0.9277518136323802, "naucs_at_1_max": 0.7069555250969274, "naucs_at_1_std": 0.03146819531411467, "naucs_at_1_diff1": 0.948883682523629, "naucs_at_3_max": 0.7542194764049448, "naucs_at_3_std": 0.14233724807946155, "naucs_at_3_diff1": 0.9448939012384912, "naucs_at_5_max": 0.7638778110599751, "naucs_at_5_std": 0.11081212928246655, "naucs_at_5_diff1": 0.9292437691902227, "naucs_at_10_max": 0.8654947821504146, "naucs_at_10_std": 0.427910709539122, "naucs_at_10_diff1": 0.927259949634811, "naucs_at_20_max": 0.8707684274521482, "naucs_at_20_std": 0.6120449232560383, "naucs_at_20_diff1": 0.9163076125059596, "naucs_at_50_max": 0.9346992729675874, "naucs_at_50_std": 0.9346992729675874, "naucs_at_50_diff1": 0.9346992729675874, "naucs_at_100_max": 0.8693985459351681, "naucs_at_100_std": 0.8693985459351681, "naucs_at_100_diff1": 1.0 }, "syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.96, "ndcg_at_3": 0.98524, "ndcg_at_5": 0.98524, "ndcg_at_10": 0.98524, "ndcg_at_20": 0.98524, "ndcg_at_50": 0.98524, "ndcg_at_100": 0.98524, "map_at_1": 0.96, "map_at_3": 0.98, "map_at_5": 0.98, "map_at_10": 0.98, "map_at_20": 0.98, "map_at_50": 0.98, "map_at_100": 0.98, "recall_at_1": 0.96, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.96, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.98, "mrr_at_5": 0.98, "mrr_at_10": 0.98, "mrr_at_20": 0.98, "mrr_at_50": 0.98, "mrr_at_100": 0.98, "naucs_at_1_max": 0.8231792717086849, "naucs_at_1_std": -0.7742763772175525, "naucs_at_1_diff1": 0.96732026143791, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "syntheticDocQA_energy_test": { "ndcg_at_1": 0.92, "ndcg_at_3": 0.95155, "ndcg_at_5": 0.95585, "ndcg_at_10": 0.95874, "ndcg_at_20": 0.95874, "ndcg_at_50": 0.96073, "ndcg_at_100": 0.96073, "map_at_1": 0.92, "map_at_3": 0.945, "map_at_5": 0.9475, "map_at_10": 0.9485, "map_at_20": 0.9485, "map_at_50": 0.94881, "map_at_100": 0.94881, "recall_at_1": 0.92, "recall_at_3": 0.97, "recall_at_5": 0.98, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32333, "precision_at_5": 0.196, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.93, "mrr_at_3": 0.95, "mrr_at_5": 0.9525, "mrr_at_10": 0.9536111111111112, "mrr_at_20": 0.9536111111111112, "mrr_at_50": 0.9539444444444445, "mrr_at_100": 0.9539444444444445, "naucs_at_1_max": 0.2832049486461252, "naucs_at_1_std": -0.673611111111113, "naucs_at_1_diff1": 0.983660130718955, "naucs_at_3_max": 0.6498599439775861, "naucs_at_3_std": -1.3478057889822568, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.540149393090577, "naucs_at_5_std": -1.7399626517273414, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.7222222222222276, "naucs_at_10_std": -1.7399626517273863, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.7222222222222276, "naucs_at_20_std": -1.7399626517273863, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "arxivqa_test_subsampled": { "ndcg_at_1": 0.866, "ndcg_at_3": 0.90455, "ndcg_at_5": 0.91117, "ndcg_at_10": 0.91638, "ndcg_at_20": 0.92041, "ndcg_at_50": 0.92254, "ndcg_at_100": 0.92416, "map_at_1": 0.866, "map_at_3": 0.89567, "map_at_5": 0.89937, "map_at_10": 0.90153, "map_at_20": 0.90263, "map_at_50": 0.90303, "map_at_100": 0.90317, "recall_at_1": 0.866, "recall_at_3": 0.93, "recall_at_5": 0.946, "recall_at_10": 0.962, "recall_at_20": 0.978, "recall_at_50": 0.988, "recall_at_100": 0.998, "precision_at_1": 0.866, "precision_at_3": 0.31, "precision_at_5": 0.1892, "precision_at_10": 0.0962, "precision_at_20": 0.0489, "precision_at_50": 0.01976, "precision_at_100": 0.00998, "mrr_at_1": 0.86, "mrr_at_3": 0.8926666666666666, "mrr_at_5": 0.8968666666666667, "mrr_at_10": 0.8986880952380951, "mrr_at_20": 0.8999191844103607, "mrr_at_50": 0.9002436444330966, "mrr_at_100": 0.9003898348214776, "naucs_at_1_max": 0.8161638583392195, "naucs_at_1_std": -0.018111189587326895, "naucs_at_1_diff1": 0.9326505902532544, "naucs_at_3_max": 0.8620114712551726, "naucs_at_3_std": 0.06083766840069296, "naucs_at_3_diff1": 0.9197012138188607, "naucs_at_5_max": 0.8768025728810052, "naucs_at_5_std": 0.2530345471521926, "naucs_at_5_diff1": 0.9255628177196795, "naucs_at_10_max": 0.8913951545530463, "naucs_at_10_std": 0.26278932625681417, "naucs_at_10_diff1": 0.9226006191950457, "naucs_at_20_max": 0.9272133095662477, "naucs_at_20_std": 0.3346490111195921, "naucs_at_20_diff1": 0.9390968508615575, "naucs_at_50_max": 0.931917211328972, "naucs_at_50_std": 0.6713352007469628, "naucs_at_50_diff1": 0.9564270152505482, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "tatdqa_test": { "ndcg_at_1": 0.71264, "ndcg_at_3": 0.8028, "ndcg_at_5": 0.81922, "ndcg_at_10": 0.83059, "ndcg_at_20": 0.83583, "ndcg_at_50": 0.83979, "ndcg_at_100": 0.84207, "map_at_1": 0.71264, "map_at_3": 0.78119, "map_at_5": 0.79039, "map_at_10": 0.79529, "map_at_20": 0.79682, "map_at_50": 0.7975, "map_at_100": 0.79771, "recall_at_1": 0.71264, "recall_at_3": 0.86513, "recall_at_5": 0.90462, "recall_at_10": 0.93864, "recall_at_20": 0.95869, "recall_at_50": 0.97813, "recall_at_100": 0.9921, "precision_at_1": 0.71264, "precision_at_3": 0.28838, "precision_at_5": 0.18092, "precision_at_10": 0.09386, "precision_at_20": 0.04793, "precision_at_50": 0.01956, "precision_at_100": 0.00992, "mrr_at_1": 0.707168894289186, "mrr_at_3": 0.7789590927501013, "mrr_at_5": 0.7882543539894699, "mrr_at_10": 0.7929583984261994, "mrr_at_20": 0.7944539276081832, "mrr_at_50": 0.7951441370914628, "mrr_at_100": 0.7953321522389292, "naucs_at_1_max": 0.267865154809863, "naucs_at_1_std": -0.09367553525202181, "naucs_at_1_diff1": 0.8523551929013733, "naucs_at_3_max": 0.3637086940124698, "naucs_at_3_std": 0.040842164765542366, "naucs_at_3_diff1": 0.7686370221763869, "naucs_at_5_max": 0.3777878526600295, "naucs_at_5_std": 0.07406870671570945, "naucs_at_5_diff1": 0.7558439662904856, "naucs_at_10_max": 0.4896273289198233, "naucs_at_10_std": 0.22965450771547347, "naucs_at_10_diff1": 0.7505849146020235, "naucs_at_20_max": 0.49880423324612594, "naucs_at_20_std": 0.3289229815026192, "naucs_at_20_diff1": 0.7088085588102754, "naucs_at_50_max": 0.6671001388202468, "naucs_at_50_std": 0.5531687668519774, "naucs_at_50_diff1": 0.79460370844467, "naucs_at_100_max": 0.7515505724064554, "naucs_at_100_std": 0.5897764215843185, "naucs_at_100_diff1": 0.7754412395814483 }, "docvqa_test_subsampled": { "ndcg_at_1": 0.54989, "ndcg_at_3": 0.6135, "ndcg_at_5": 0.63058, "ndcg_at_10": 0.65304, "ndcg_at_20": 0.665, "ndcg_at_50": 0.67275, "ndcg_at_100": 0.67814, "map_at_1": 0.54989, "map_at_3": 0.59793, "map_at_5": 0.60724, "map_at_10": 0.61622, "map_at_20": 0.6196, "map_at_50": 0.62077, "map_at_100": 0.62124, "recall_at_1": 0.54989, "recall_at_3": 0.65854, "recall_at_5": 0.70067, "recall_at_10": 0.77162, "recall_at_20": 0.81818, "recall_at_50": 0.85809, "recall_at_100": 0.89135, "precision_at_1": 0.54989, "precision_at_3": 0.21951, "precision_at_5": 0.14013, "precision_at_10": 0.07716, "precision_at_20": 0.04091, "precision_at_50": 0.01716, "precision_at_100": 0.00891, "mrr_at_1": 0.5521064301552107, "mrr_at_3": 0.5931263858093129, "mrr_at_5": 0.6059866962305985, "mrr_at_10": 0.6145892725161016, "mrr_at_20": 0.6172970783835527, "mrr_at_50": 0.6190029838259105, "mrr_at_100": 0.6194633017577265, "naucs_at_1_max": 0.3120684712170286, "naucs_at_1_std": 0.7079281345869217, "naucs_at_1_diff1": 0.917948356572515, "naucs_at_3_max": 0.24008650595118855, "naucs_at_3_std": 0.8128519605964449, "naucs_at_3_diff1": 0.8473099212945505, "naucs_at_5_max": 0.16205532036953102, "naucs_at_5_std": 0.8560263843379905, "naucs_at_5_diff1": 0.824925647993079, "naucs_at_10_max": 0.012769751585812625, "naucs_at_10_std": 0.9065613989192631, "naucs_at_10_diff1": 0.8065057384718499, "naucs_at_20_max": -0.06721876143851727, "naucs_at_20_std": 0.9197324080389992, "naucs_at_20_diff1": 0.7911368704615722, "naucs_at_50_max": -0.17369534057340794, "naucs_at_50_std": 0.9264211427158648, "naucs_at_50_diff1": 0.7869124563082568, "naucs_at_100_max": -0.22153475665071373, "naucs_at_100_std": 0.9125411363750677, "naucs_at_100_diff1": 0.7666876762968439 } }