@Test public void shouldDetectThatTheSame() throws Exception { SearchResultItem item1 = new SearchResultItem(); setValues(item1, "1", "poster", "group", Instant.now()); SearchResultItem item2 = new SearchResultItem(); setValues(item2, "2", "poster", "group", Instant.now()); SearchResultItem item3 = new SearchResultItem(); setValues(item3, "3", "poster", "group", Instant.now()); DuplicateDetectionResult result = testee.detectDuplicates(Sets.newHashSet(item1, item2, item3)); assertThat(result.getDuplicateGroups().size()).isEqualTo(1); List<SearchResultItem> items = new ArrayList<>(result.getDuplicateGroups().get(0)); assertThat(items.get(0).getDuplicateIdentifier()).isEqualTo(items.get(1).getDuplicateIdentifier()).isEqualTo(items.get(2).getDuplicateIdentifier()); }
public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); } }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }  }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }  DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results); }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }  DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results);  }