Use different algorithms for different documents ratios
@@ -94,32 +94,59 @@ where
let before = Instant::now();
+ let docidslen = docids.len() as f32;
let mut bare_matches = Vec::new();
mk_arena!(arena);
for ((query, input, distance), matches) in queries {
let postings_list_view = PostingsListView::original(Rc::from(input), Rc::new(matches));
+ let pllen = postings_list_view.len() as f32;
+
+ if docidslen / pllen >= 0.8 {
+ let mut offset = 0;
+ for matches in postings_list_view.linear_group_by_key(|m| m.document_id) {
+ let document_id = matches[0].document_id;
+ if docids.contains(&document_id) {
+ let range = postings_list_view.range(offset, matches.len());
+ let posting_list_index = arena.add(range);
+
+ let bare_match = BareMatch {
+ document_id,
+ query_index: query.id,
+ distance,
+ is_exact: true, // TODO where can I find this info?
+ postings_list: posting_list_index,
+ };
+
+ bare_matches.push(bare_match);
+ }
+
+ offset += matches.len();
+ }
+
+ } else {
+
let mut offset = 0;
for id in docids.as_slice() {
let di = DocIndex { document_id: *id, ..DocIndex::default() };
let pos = exponential_search(&postings_list_view[offset..], &di).unwrap_or_else(|x| x);
- let group = postings_list_view[offset + pos..]
+ offset += pos;
+
+ let group = postings_list_view[offset..]
.linear_group_by_key(|m| m.document_id)
.next()
.filter(|matches| matches[0].document_id == *id);
- offset += pos;
-
if let Some(matches) = group {
- let range = postings_list_view.range(pos, matches.len());
+ let range = postings_list_view.range(offset, matches.len());
let posting_list_index = arena.add(range);
let bare_match = BareMatch {
document_id: *id,
query_index: query.id,
- distance: distance,
+ distance,
is_exact: true, // TODO where can I find this info?
postings_list: posting_list_index,
};
@@ -128,6 +155,7 @@ where
}
}
}
+ }
println!("matches cleaned in {:.02?}", before.elapsed());
