TextPopulator { void populate(File dataFile, File indexDir) throws IOException { try (Closer closer = Closer.create()) { Iterable<CSVRecord> csvRecords = closer.register(CSVParser.parse(dataFile, UTF_8, FORMAT)); final FSDirectory dir = closer.register(FSDirectory.open(indexDir)); final DirectoryReader reader = closer.register(DirectoryReader.open(dir)); final IndexSearcher searcher = new IndexSearcher(reader); for (CSVRecord record : csvRecords) { String blobId = record.get(BLOB_ID); String jcrPath = record.get(JCR_PATH); if (!textWriter.isProcessed(blobId)) { String text = getText(reader, searcher, jcrPath); stats.processed++; if (text == null) { stats.errored++; } else if (ERROR_TEXT.equals(text)) { textWriter.markError(blobId); stats.errored++; } else if (text.length() == 0) { textWriter.markEmpty(blobId); stats.empty++; } else { textWriter.write(blobId, text); stats.parsed++; } } else { stats.ignored++; } stats.readAndDumpStatsIfRequired(jcrPath); } log.info(stats.toString()); } } TextPopulator(TextWriter textWriter);   }
@Test public void simpleTest() throws Exception { setupCSV("/sentence", "/para"); textPopulator.populate(csv, indexDir); assertEquals("Incorrect binaries processed", 2, stats.processed); textPopulator.populate(csv, indexDir); assertEquals("Repeated call for already processed stuff shouldn't process anything more", 2, stats.ignored); assertConsistentStatsAndWriter(); assertStatsInvariants(); }
@Test public void untrimmedText() throws Exception { setupCSV("/untrimmed"); textPopulator.populate(csv, indexDir); assertEquals("Store generation didn't trim data", "untrimmed", textWriter.data.get(FakeBinaryResourceProvider.getBlobId("/untrimmed"))); assertConsistentStatsAndWriter(); assertStatsInvariants(); }
@Test public void indexedError() throws Exception { setupCSV("/error"); textPopulator.populate(csv, indexDir); assertEquals("Indexed data reporting errored extraction not marked as error", 1, stats.errored); textPopulator.populate(csv, indexDir); assertEquals("Repeated run for indexed error shouldn't get processed again", 1, stats.ignored); assertConsistentStatsAndWriter(); assertStatsInvariants(); }
@Test public void indexedEmpty() throws Exception { setupCSV("/empty"); textPopulator.populate(csv, indexDir); assertEquals("Indexed data for empty extraction not marked as empty", 1, stats.empty); textPopulator.populate(csv, indexDir); assertEquals("Repeated run for empty extraction shouldn't get processed again", 1, stats.ignored); assertConsistentStatsAndWriter(); assertStatsInvariants(); }
@Test public void indexedUntrimmedEmpty() throws Exception { setupCSV("/untrimmed-empty"); textPopulator.populate(csv, indexDir); assertEquals("Indexed data for untrimmed empty extraction not marked as empty", 1, stats.empty); textPopulator.populate(csv, indexDir); assertEquals("Repeated run for untrimmed empty extraction shouldn't get processed again", 1, stats.ignored); assertConsistentStatsAndWriter(); assertStatsInvariants(); }
@Test public void multiFTField() throws Exception { setupCSV("/multi"); textPopulator.populate(csv, indexDir); assertEquals("Multi FT field in a doc not marked as error", 1, stats.errored); textPopulator.populate(csv, indexDir); assertEquals("Repeated run for multi FT error should get processed again", 0, stats.ignored); assertStatsInvariants(); }
@Test public void indexHasDocumentButNotData() throws Exception { setupCSV("/null"); textPopulator.populate(csv, indexDir); assertEquals("No FT field in a doc not marked as error", 1, stats.errored); textPopulator.populate(csv, indexDir); assertEquals("Repeated run for no FT error should get processed again", 0, stats.ignored); assertStatsInvariants(); }
@Test public void indexDoesNotHaveDocument() throws Exception { setupCSV("/somethingRandom"); textPopulator.populate(csv, indexDir); assertEquals("No indexed doc not marked as error", 1, stats.errored); textPopulator.populate(csv, indexDir); assertEquals("Repeated run for no indexed doc error should get processed again", 0, stats.ignored); assertStatsInvariants(); }