@Test public void multiple() throws MalformedURLException { byte[] stream = "<html><body><test><o>value1</o><t>key1</t></test><test><o>value2</o><t>key2</t></test><test><o>value3</o></test></body></html>" .getBytes(); Blob blob = new Blob(new BlobMetadata().setUri(Uri.create("http: DefaultDataExtractor extractor = new DefaultDataExtractor(); DocumentStores.add(0, extractor.getName(), documentStore); FieldExtractor dateField = new FieldExtractor(); dateField.setName("date"); Extractor dateFieldExtractor = new Extractor(); dateFieldExtractor.setType("xpath"); dateFieldExtractor.setSource(SourceType.BODY); dateFieldExtractor.setValue("/test/o/text()"); dateField.setExtractor(dateFieldExtractor); FieldExtractor keyField = new FieldExtractor(); keyField.setName("key"); Extractor keyFieldExtractor = new Extractor(); keyFieldExtractor.setType("xpath"); keyFieldExtractor.setSource(SourceType.BODY); keyFieldExtractor.setValue(" keyField.setExtractor(keyFieldExtractor); extractor.setFields(Arrays.asList(dateField, keyField)); Extractor multiple = new Extractor(); multiple.setType("xpath"); multiple.setSource(SourceType.BODY); multiple.setValue("/html/body/test"); extractor.setMultiple(multiple); extractorService.extractThenFormatThenStore(0, new HashMap<>(), blob, extractor); Document data1 = new Document(); data1.put("date", Arrays.asList("value1")); data1.put("key", Arrays.asList("key1")); Document data2 = new Document(); data2.put("date", Arrays.asList("value2")); data2.put("key", Arrays.asList("key2")); Document data3 = new Document(); data3.put("date", Arrays.asList("value3")); Mockito.verify(documentStore).save(Arrays.asList(data1, data2, data3)); }
public List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor) { List<Document> documents = extractThenFormat(cachedSelectors, blob, extractor); if (documents != null) { DocumentStores.get(jobId, extractor.getName()).get().save(documents); } return documents; }
ExtractorService { public List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor) { List<Document> documents = extractThenFormat(cachedSelectors, blob, extractor); if (documents != null) { DocumentStores.get(jobId, extractor.getName()).get().save(documents); } return documents; } }
ExtractorService { public List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor) { List<Document> documents = extractThenFormat(cachedSelectors, blob, extractor); if (documents != null) { DocumentStores.get(jobId, extractor.getName()).get().save(documents); } return documents; } @Inject  ExtractorService(ScriptingService scriptingService); }
ExtractorService { public List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor) { List<Document> documents = extractThenFormat(cachedSelectors, blob, extractor); if (documents != null) { DocumentStores.get(jobId, extractor.getName()).get().save(documents); } return documents; } @Inject  ExtractorService(ScriptingService scriptingService); Pair<Set<Link>, Set<Link>> extractAndFilterOutlinks(Job job, Uri uri, Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor ol); Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor); List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor); List<Document> extractThenFormat(Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor); }
ExtractorService { public List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor) { List<Document> documents = extractThenFormat(cachedSelectors, blob, extractor); if (documents != null) { DocumentStores.get(jobId, extractor.getName()).get().save(documents); } return documents; } @Inject  ExtractorService(ScriptingService scriptingService); Pair<Set<Link>, Set<Link>> extractAndFilterOutlinks(Job job, Uri uri, Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor ol); Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor); List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor); List<Document> extractThenFormat(Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor);  }