@Test public void testExtraction00() throws Exception { String html = loadArticle("aljazeera1"); String url = "https: Document document = Jsoup.parse(html); assertEquals(url, UrlExtractor.extract(url, document)); assertEquals("https: }
private static Set<String> extract(Document document) { Set<String> canonicalUrls = new HashSet<>(); if (document == null) { return canonicalUrls; } Elements elements = document.select("meta[property=og:url]"); elements.forEach(element -> { String attr = element.attr("content"); if (attr != null) { canonicalUrls.add(attr); } }); elements = document.select("link[rel=canonical]"); elements.forEach(element -> { String attr = element.attr("href"); if (attr != null) { canonicalUrls.add(attr); } }); return canonicalUrls.stream() .filter(UrlExtractor::isAbsolute) .collect(Collectors.toSet()); }
UrlExtractor { private static Set<String> extract(Document document) { Set<String> canonicalUrls = new HashSet<>(); if (document == null) { return canonicalUrls; } Elements elements = document.select("meta[property=og:url]"); elements.forEach(element -> { String attr = element.attr("content"); if (attr != null) { canonicalUrls.add(attr); } }); elements = document.select("link[rel=canonical]"); elements.forEach(element -> { String attr = element.attr("href"); if (attr != null) { canonicalUrls.add(attr); } }); return canonicalUrls.stream() .filter(UrlExtractor::isAbsolute) .collect(Collectors.toSet()); } }
UrlExtractor { private static Set<String> extract(Document document) { Set<String> canonicalUrls = new HashSet<>(); if (document == null) { return canonicalUrls; } Elements elements = document.select("meta[property=og:url]"); elements.forEach(element -> { String attr = element.attr("content"); if (attr != null) { canonicalUrls.add(attr); } }); elements = document.select("link[rel=canonical]"); elements.forEach(element -> { String attr = element.attr("href"); if (attr != null) { canonicalUrls.add(attr); } }); return canonicalUrls.stream() .filter(UrlExtractor::isAbsolute) .collect(Collectors.toSet()); }  }
UrlExtractor { private static Set<String> extract(Document document) { Set<String> canonicalUrls = new HashSet<>(); if (document == null) { return canonicalUrls; } Elements elements = document.select("meta[property=og:url]"); elements.forEach(element -> { String attr = element.attr("content"); if (attr != null) { canonicalUrls.add(attr); } }); elements = document.select("link[rel=canonical]"); elements.forEach(element -> { String attr = element.attr("href"); if (attr != null) { canonicalUrls.add(attr); } }); return canonicalUrls.stream() .filter(UrlExtractor::isAbsolute) .collect(Collectors.toSet()); }  static String extract(String url, Document document); }
UrlExtractor { private static Set<String> extract(Document document) { Set<String> canonicalUrls = new HashSet<>(); if (document == null) { return canonicalUrls; } Elements elements = document.select("meta[property=og:url]"); elements.forEach(element -> { String attr = element.attr("content"); if (attr != null) { canonicalUrls.add(attr); } }); elements = document.select("link[rel=canonical]"); elements.forEach(element -> { String attr = element.attr("href"); if (attr != null) { canonicalUrls.add(attr); } }); return canonicalUrls.stream() .filter(UrlExtractor::isAbsolute) .collect(Collectors.toSet()); }  static String extract(String url, Document document);  }