HeuristicHashCitationMatcher { public HeuristicHashMatchingResult matchCitations(JavaPairRDD<String, MatchableEntity> citations, JavaPairRDD<String, MatchableEntity> documents, boolean needUnmatched) { JavaPairRDD<String, String> citationHashIdPairs = generateHashIdPairs(citations, citationHasher); citationHashIdPairs = citationHashIdPairs.partitionBy(new HashPartitioner(citationHashIdPairs.partitions().size())); JavaPairRDD<String, String> documentHashIdPairs = generateHashIdPairs(documents, documentHasher); documentHashIdPairs = documentHashIdPairs.partitionBy(new HashPartitioner(documentHashIdPairs.partitions().size())); JavaPairRDD<String, Long> invalidHashes = invalidHashExtractor.extractInvalidHashes(citationHashIdPairs, documentHashIdPairs, maxHashBucketSize); citationHashIdPairs = citationHashIdPairs.subtractByKey(invalidHashes); documentHashIdPairs = documentHashIdPairs.subtractByKey(invalidHashes); JavaPairRDD<String, String> citationDocumentIdPairs = citationHashIdPairs.join(documentHashIdPairs).mapToPair(cd->cd._2()).distinct(); JavaPairRDD<String, MatchableEntity> unmatchedCitations = null; if (needUnmatched) { unmatchedCitations = citations.subtractByKey(citationDocumentIdPairs); } return new HeuristicHashMatchingResult(citationDocumentIdPairs, unmatchedCitations); } HeuristicHashCitationMatcher(MatchableEntityHasher citationHasher, MatchableEntityHasher documentHasher, long maxHashBucketSize); HeuristicHashMatchingResult matchCitations(JavaPairRDD<String, MatchableEntity> citations, JavaPairRDD<String, MatchableEntity> documents, 
            boolean needUnmatched);  }
@Test public void matchCitations_4() throws IOException { String citationPath = "src/test/resources/heuristic/unmatched-3"; String documentPath = "src/test/resources/heuristic/documents"; HeuristicHashCitationMatcher heuristicMatcher = new HeuristicHashCitationMatcher( createMatchableEntityHasher(new CitationNameYearHashGenerator()), createMatchableEntityHasher(new DocumentNameYearHashGenerator()), 10000); JavaPairRDD<String, MatchableEntity> citations = loadEntities(sparkContext, citationPath); JavaPairRDD<String, MatchableEntity> documents = loadEntities(sparkContext, documentPath); HeuristicHashMatchingResult matchedResult = heuristicMatcher.matchCitations(citations, documents, false); assertMatchedCitations(matchedResult.getCitDocIdPairs().collect(), "src/test/resources/heuristic/heuristic-out-4"); assertNull(matchedResult.getUnmatchedCitations()); }
@Test public void matchCitations_1() throws IOException { String citationPath = "src/test/resources/heuristic/citations"; String documentPath = "src/test/resources/heuristic/documents"; HeuristicHashCitationMatcher heuristicMatcher = new HeuristicHashCitationMatcher( createMatchableEntityHasher(new CitationNameYearPagesHashGenerator()), createMatchableEntityHasher(new DocumentNameYearPagesHashGenerator()), 10000); JavaPairRDD<String, MatchableEntity> citations = loadEntities(sparkContext, citationPath); JavaPairRDD<String, MatchableEntity> documents = loadEntities(sparkContext, documentPath); HeuristicHashMatchingResult matchedResult = heuristicMatcher.matchCitations(citations, documents, true); assertMatchedCitations(matchedResult.getCitDocIdPairs().collect(), "src/test/resources/heuristic/heuristic-out-1"); assertUnmatchedCitations(matchedResult.getUnmatchedCitations().collect(), "src/test/resources/heuristic/unmatched-1"); }
@Test public void matchCitations_2() throws IOException { String citationPath = "src/test/resources/heuristic/unmatched-1"; String documentPath = "src/test/resources/heuristic/documents"; HeuristicHashCitationMatcher heuristicMatcher = new HeuristicHashCitationMatcher( createMatchableEntityHasher(new CitationNameYearPagesHashGenerator()), createMatchableEntityHasher(new DocumentNameYearNumNumHashGenerator()), 10000); JavaPairRDD<String, MatchableEntity> citations = loadEntities(sparkContext, citationPath); JavaPairRDD<String, MatchableEntity> documents = loadEntities(sparkContext, documentPath); HeuristicHashMatchingResult matchedResult = heuristicMatcher.matchCitations(citations, documents, true); assertMatchedCitations(matchedResult.getCitDocIdPairs().collect(), "src/test/resources/heuristic/heuristic-out-2"); assertUnmatchedCitations(matchedResult.getUnmatchedCitations().collect(), "src/test/resources/heuristic/unmatched-2"); }
@Test public void matchCitations_3() throws IOException { String citationPath = "src/test/resources/heuristic/unmatched-2"; String documentPath = "src/test/resources/heuristic/documents"; HeuristicHashCitationMatcher heuristicMatcher = new HeuristicHashCitationMatcher( createMatchableEntityHasher(new CitationNameYearHashGenerator()), createMatchableEntityHasher(new DocumentNameYearStrictHashGenerator()), 10000); JavaPairRDD<String, MatchableEntity> citations = loadEntities(sparkContext, citationPath); JavaPairRDD<String, MatchableEntity> documents = loadEntities(sparkContext, documentPath); HeuristicHashMatchingResult matchedResult = heuristicMatcher.matchCitations(citations, documents, true); assertMatchedCitations(matchedResult.getCitDocIdPairs().collect(), "src/test/resources/heuristic/heuristic-out-3"); assertUnmatchedCitations(matchedResult.getUnmatchedCitations().collect(), "src/test/resources/heuristic/unmatched-3"); }