@Test public void htmlParsing() { HashMap<String, String> config = Maps.newHashMap(); HtmlAnalysisResult result = PageAnalyzer.analyze(config, "https: assertEquals("Bloomberg.com", result.getTitle()); assertEquals(33, result.getMetaValues().size()); assertTrue(result.getMetaValues().contains("<meta property=\"og:url\" content=\"https: assertEquals(361, result.getLinks().size()); assertTrue(result.getLinks().contains("https: }
public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } } }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }  }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }  static HtmlAnalysisResult analyze(Map<String, String> config, String url); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html, Integer status, Map<String, List<String>> headers); static void parseRobotsTxt(String userAgent, String robotsUrl, String robotsTxt, HtmlAnalysisResult result); }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }  static HtmlAnalysisResult analyze(Map<String, String> config, String url); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html, Integer status, Map<String, List<String>> headers); static void parseRobotsTxt(String userAgent, String robotsUrl, String robotsTxt, HtmlAnalysisResult result); static final String CONFIG_USER_AGENT; static final String CONFIG_ANALYZE_ROBOTS_TXT; }