@Test public void shouldWorkWithOneIndexerProvidingGroupAndPosterAndOneNot() throws Exception { SearchResultItem item1 = new SearchResultItem(); setValues(item1, "Indexer1", "chuck@norris.com", "alt.binaries.triballs", Instant.ofEpochSecond(1447928064)); item1.setUsenetDate(item1.getPubDate()); item1.setSize(11565521038L); SearchResultItem item2 = new SearchResultItem(); setValues(item2, "Indexer1", "moovee@4u.tv (moovee)", "alt.binaries.moovee", Instant.ofEpochSecond(1447930279)); item2.setUsenetDate(item2.getPubDate()); item2.setSize(12100381412L); SearchResultItem item3 = new SearchResultItem(); setValues(item3, "Indexer1", "chuck@norris.com", "alt.binaries.triballs", Instant.ofEpochSecond(1447927640)); item3.setUsenetDate(item3.getPubDate()); item3.setSize(11565520866L); SearchResultItem item4 = new SearchResultItem(); setValues(item4, "Indexer1", "moovee@4u.tv (moovee)", "alt.binaries.moovee", Instant.ofEpochSecond(1447930279)); item4.setUsenetDate(item4.getPubDate()); item4.setSize(12100382514L); SearchResultItem item5 = new SearchResultItem(); setValues(item5, "Indexer2", null, null, Instant.ofEpochSecond(1447973616)); item5.setSize(12096598793L); SearchResultItem item6 = new SearchResultItem(); setValues(item6, "Indexer2", null, null, Instant.ofEpochSecond(1447945386)); item6.setSize(12078310717L); SearchResultItem item7 = new SearchResultItem(); setValues(item7, "Indexer2", null, null, Instant.ofEpochSecond(1447930475)); item7.setSize(12099830939L); SearchResultItem item8 = new SearchResultItem(); setValues(item8, "Indexer2", null, null, Instant.ofEpochSecond(1447928088)); item8.setSize(11566348892L); HashSet<SearchResultItem> results = Sets.newHashSet(item1, item2, item3, item4, item5, item6, item7, item8); DuplicateDetectionResult result = testee.detectDuplicates(results); assertThat(result.getDuplicateGroups().size()).isEqualTo(6); assertThat(result.getDuplicateGroups().get(0).size()).isEqualTo(1); assertThat(result.getDuplicateGroups().get(1).size()).isEqualTo(1); assertThat(result.getDuplicateGroups().get(2).size()).isEqualTo(2); assertThat(result.getDuplicateGroups().get(3).size()).isEqualTo(1); assertThat(result.getDuplicateGroups().get(4).size()).isEqualTo(2); assertThat(result.getDuplicateGroups().get(5).size()).isEqualTo(1); }
public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); } }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }  }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }  DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results); }
DuplicateDetector { public DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results) { Map<String, List<SearchResultItem>> groupedByTitle = results.stream().collect(Collectors.groupingBy(x -> x.getTitle().toLowerCase().replaceAll("[ .\\-_]", ""))); Multiset<Indexer> countUniqueResultsPerIndexer = HashMultiset.create(); List<LinkedHashSet<SearchResultItem>> duplicateGroups = new ArrayList<>(); int countDetectedDuplicates = 0; for (List<SearchResultItem> titleGroup : groupedByTitle.values()) { titleGroup = titleGroup.stream().sorted(Comparator.comparing(SearchResultItem::getBestDate).reversed()).collect(Collectors.toList()); List<LinkedHashSet<SearchResultItem>> listOfBuckets = new ArrayList<>(); listOfBuckets.add(new LinkedHashSet<>(newArrayList(titleGroup.get(0)))); for (int i = 1; i < titleGroup.size(); i++) { SearchResultItem searchResultItem = titleGroup.get(i); boolean foundBucket = false; for (LinkedHashSet<SearchResultItem> bucket : listOfBuckets) { if (bucket.stream().map(SearchResultItem::getIndexer).collect(Collectors.toList()).contains(searchResultItem.getIndexer())) { continue; } for (SearchResultItem other : bucket) { boolean same = testForSameness(searchResultItem, other); if (same) { foundBucket = true; bucket.add(searchResultItem); countDetectedDuplicates++; break; } } if (foundBucket) { break; } } if (!foundBucket) { listOfBuckets.add(new LinkedHashSet<>(newArrayList(searchResultItem))); } } LinkedHashSet<SearchResultItem> lastBucket = Iterables.getLast(listOfBuckets); if (lastBucket.size() == 1) { countUniqueResultsPerIndexer.add(lastBucket.iterator().next().getIndexer()); } duplicateGroups.addAll(listOfBuckets); } int duplicateIdentifier = 0; for (LinkedHashSet<SearchResultItem> group : duplicateGroups) { for (SearchResultItem x : group) { x.setDuplicateIdentifier(duplicateIdentifier); } duplicateIdentifier++; } logger.debug("Duplicate detection for {} search results found {} duplicates", results.size(), countDetectedDuplicates); return new DuplicateDetectionResult(duplicateGroups, countUniqueResultsPerIndexer, countDetectedDuplicates); }  DuplicateDetectionResult detectDuplicates(Set<SearchResultItem> results);  }