@Test public void headersAndStatus() { Map<String, List<String>> headers = Maps.newHashMap(); headers.put("Etag", Lists.newArrayList("c1dc8d7be85325149", "ed5fc4d62b84752")); headers.put("Date", Lists.newArrayList("Wed, 11 Jan 2017 13:00:18 GMT")); HashMap<String, String> config = Maps.newHashMap(); HtmlAnalysisResult result = PageAnalyzer.analyze(config, "http: assertEquals(new Integer(200), result.getHttpStatus()); assertEquals(2, result.getHeaders().size()); assertEquals("c1dc8d7be85325149\ned5fc4d62b84752", result.getHeaders().get("Etag")); assertEquals("Wed, 11 Jan 2017 13:00:18 GMT", result.getHeaders().get("Date")); }
public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } } }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }  }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }  static HtmlAnalysisResult analyze(Map<String, String> config, String url); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html, Integer status, Map<String, List<String>> headers); static void parseRobotsTxt(String userAgent, String robotsUrl, String robotsTxt, HtmlAnalysisResult result); }
PageAnalyzer { public static HtmlAnalysisResult analyze(Map<String, String> config, String url) { try { String userAgent = config.getOrDefault(CONFIG_USER_AGENT, DEFAULT_USER_AGENT); HttpResponse<String> response = Unirest.get(url) .header("User-Agent", userAgent) .asString(); return analyze(config, url, response.getBody(), response.getStatus(), response.getHeaders()); } catch (UnirestException e) { throw new RuntimeException(e); } }  static HtmlAnalysisResult analyze(Map<String, String> config, String url); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html); static HtmlAnalysisResult analyze(Map<String, String> config, String url, String html, Integer status, Map<String, List<String>> headers); static void parseRobotsTxt(String userAgent, String robotsUrl, String robotsTxt, HtmlAnalysisResult result); static final String CONFIG_USER_AGENT; static final String CONFIG_ANALYZE_ROBOTS_TXT; }