@Test public void outlinks() throws MalformedURLException { byte[] stream = "<html><body><a href='http: Blob blob = new Blob(new BlobMetadata().setUri(Uri.create("http: OutlinkExtractor extractor = new OutlinkExtractor("_default"); Set<Link> links = extractorService.extractOutlinks(new HashMap<>(), blob, extractor); Assertions.assertThat(links).containsOnly(new Link().setText("Relative").setUri(Uri.create("http: new Link().setText("Absolute").setUri(Uri.create("http: }
public Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor) { List<Link> outlinks = DefaultDataExtractor.extract(cachedSelectors, blob, null, extractor.getExtractor(), new DataConverter<XElement, Link>() { public Link convert(XElement element) { Link link = new Link(); String uri = element.getElement().absUrl("href"); link.setUri(StringUtils.isNotBlank(uri) ? Uri.create(uri) : null); String rel = element.getElement().attr("rel"); link.setRel(StringUtils.isNotBlank(rel) ? rel : null); String title = element.getElement().attr("title"); link.setTitle(StringUtils.isNotBlank(title) ? title : null); String text = element.getElement().ownText(); link.setText(StringUtils.isNotBlank(text) ? text : null); return link; } }); if (outlinks != null && !outlinks.isEmpty()) { outlinks = (List<Link>) DefaultDataExtractor.format(scriptingService, blob.getMetadata(), extractor, outlinks); } return new HashSet<>(outlinks); }
ExtractorService { public Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor) { List<Link> outlinks = DefaultDataExtractor.extract(cachedSelectors, blob, null, extractor.getExtractor(), new DataConverter<XElement, Link>() { public Link convert(XElement element) { Link link = new Link(); String uri = element.getElement().absUrl("href"); link.setUri(StringUtils.isNotBlank(uri) ? Uri.create(uri) : null); String rel = element.getElement().attr("rel"); link.setRel(StringUtils.isNotBlank(rel) ? rel : null); String title = element.getElement().attr("title"); link.setTitle(StringUtils.isNotBlank(title) ? title : null); String text = element.getElement().ownText(); link.setText(StringUtils.isNotBlank(text) ? text : null); return link; } }); if (outlinks != null && !outlinks.isEmpty()) { outlinks = (List<Link>) DefaultDataExtractor.format(scriptingService, blob.getMetadata(), extractor, outlinks); } return new HashSet<>(outlinks); } }
ExtractorService { public Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor) { List<Link> outlinks = DefaultDataExtractor.extract(cachedSelectors, blob, null, extractor.getExtractor(), new DataConverter<XElement, Link>() { public Link convert(XElement element) { Link link = new Link(); String uri = element.getElement().absUrl("href"); link.setUri(StringUtils.isNotBlank(uri) ? Uri.create(uri) : null); String rel = element.getElement().attr("rel"); link.setRel(StringUtils.isNotBlank(rel) ? rel : null); String title = element.getElement().attr("title"); link.setTitle(StringUtils.isNotBlank(title) ? title : null); String text = element.getElement().ownText(); link.setText(StringUtils.isNotBlank(text) ? text : null); return link; } }); if (outlinks != null && !outlinks.isEmpty()) { outlinks = (List<Link>) DefaultDataExtractor.format(scriptingService, blob.getMetadata(), extractor, outlinks); } return new HashSet<>(outlinks); } @Inject  ExtractorService(ScriptingService scriptingService); }
ExtractorService { public Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor) { List<Link> outlinks = DefaultDataExtractor.extract(cachedSelectors, blob, null, extractor.getExtractor(), new DataConverter<XElement, Link>() { public Link convert(XElement element) { Link link = new Link(); String uri = element.getElement().absUrl("href"); link.setUri(StringUtils.isNotBlank(uri) ? Uri.create(uri) : null); String rel = element.getElement().attr("rel"); link.setRel(StringUtils.isNotBlank(rel) ? rel : null); String title = element.getElement().attr("title"); link.setTitle(StringUtils.isNotBlank(title) ? title : null); String text = element.getElement().ownText(); link.setText(StringUtils.isNotBlank(text) ? text : null); return link; } }); if (outlinks != null && !outlinks.isEmpty()) { outlinks = (List<Link>) DefaultDataExtractor.format(scriptingService, blob.getMetadata(), extractor, outlinks); } return new HashSet<>(outlinks); } @Inject  ExtractorService(ScriptingService scriptingService); Pair<Set<Link>, Set<Link>> extractAndFilterOutlinks(Job job, Uri uri, Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor ol); Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor); List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor); List<Document> extractThenFormat(Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor); }
ExtractorService { public Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor) { List<Link> outlinks = DefaultDataExtractor.extract(cachedSelectors, blob, null, extractor.getExtractor(), new DataConverter<XElement, Link>() { public Link convert(XElement element) { Link link = new Link(); String uri = element.getElement().absUrl("href"); link.setUri(StringUtils.isNotBlank(uri) ? Uri.create(uri) : null); String rel = element.getElement().attr("rel"); link.setRel(StringUtils.isNotBlank(rel) ? rel : null); String title = element.getElement().attr("title"); link.setTitle(StringUtils.isNotBlank(title) ? title : null); String text = element.getElement().ownText(); link.setText(StringUtils.isNotBlank(text) ? text : null); return link; } }); if (outlinks != null && !outlinks.isEmpty()) { outlinks = (List<Link>) DefaultDataExtractor.format(scriptingService, blob.getMetadata(), extractor, outlinks); } return new HashSet<>(outlinks); } @Inject  ExtractorService(ScriptingService scriptingService); Pair<Set<Link>, Set<Link>> extractAndFilterOutlinks(Job job, Uri uri, Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor ol); Set<Link> extractOutlinks(Map<String, Instance<?>> cachedSelectors, Blob blob, OutlinkExtractor extractor); List<Document> extractThenFormatThenStore(long jobId, Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor); List<Document> extractThenFormat(Map<String, Instance<?>> cachedSelectors, Blob blob, DataExtractor extractor);  }