HttpClientDownloader extends AbstractDownloader { @Override public Page download(Request request, Task task) { if (task == null || task.getSite() == null) { throw new NullPointerException("task or site can not be null"); } CloseableHttpResponse httpResponse = null; CloseableHttpClient httpClient = getHttpClient(task.getSite()); Proxy proxy = proxyProvider != null ? proxyProvider.getProxy(task) : null; HttpClientRequestContext requestContext = httpUriRequestConverter.convert(request, task.getSite(), proxy); Page page = Page.fail(); try { httpResponse = httpClient.execute(requestContext.getHttpUriRequest(), requestContext.getHttpClientContext()); page = handleResponse(request, request.getCharset() != null ? request.getCharset() : task.getSite().getCharset(), httpResponse, task); onSuccess(request); logger.info("downloading page success {}", request.getUrl()); return page; } catch (IOException e) { logger.warn("download page {} error", request.getUrl(), e); onError(request); return page; } finally { if (httpResponse != null) { EntityUtils.consumeQuietly(httpResponse.getEntity()); } if (proxyProvider != null && proxy != null) { proxyProvider.returnProxy(proxy, page, task); } } }  void setHttpUriRequestConverter(HttpUriRequestConverter httpUriRequestConverter); void setProxyProvider(ProxyProvider proxyProvider); @Override Page download(Request request, Task task); @Override void setThread(int thread);  }
@Test public void testDownloader() { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Html html = httpClientDownloader.download("https: assertTrue(!html.getFirstSourceText().isEmpty()); }
@Test(expected = IllegalArgumentException.class) public void testDownloaderInIllegalUrl() throws UnsupportedEncodingException { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); httpClientDownloader.download("http: }
@Test public void test_download_fail() { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Task task = Site.me().setDomain("localhost").setCycleRetryTimes(5).toTask(); Request request = new Request(PAGE_ALWAYS_NOT_EXISTS); Page page = httpClientDownloader.download(request, task); assertThat(page.isDownloadSuccess()).isFalse(); }
@Test public void test_set_request_cookie() throws Exception { HttpServer server = httpServer(13423); server.get(eq(cookie("cookie"), "cookie-webmagic")).response("ok"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setUrl("http: request.addCookie("cookie","cookie-webmagic"); Page page = httpClientDownloader.download(request, Site.me().toTask()); assertThat(page.getRawText()).isEqualTo("ok"); } }); }
@Test public void test_disableCookieManagement() throws Exception { HttpServer server = httpServer(13423); server.get(not(eq(cookie("cookie"), "cookie-webmagic"))).response("ok"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setUrl("http: request.addCookie("cookie","cookie-webmagic"); Page page = httpClientDownloader.download(request, Site.me().setDisableCookieManagement(true).toTask()); assertThat(page.getRawText()).isEqualTo("ok"); } }); }
@Test public void test_set_request_header() throws Exception { HttpServer server = httpServer(13423); server.get(eq(header("header"), "header-webmagic")).response("ok"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setUrl("http: request.addHeader("header","header-webmagic"); Page page = httpClientDownloader.download(request, Site.me().toTask()); assertThat(page.getRawText()).isEqualTo("ok"); } }); }
@Test public void test_set_site_header() throws Exception { HttpServer server = httpServer(13423); server.get(eq(header("header"), "header-webmagic")).response("ok"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setUrl("http: Page page = httpClientDownloader.download(request, Site.me().addHeader("header","header-webmagic").toTask()); assertThat(page.getRawText()).isEqualTo("ok"); } }); }
@Test public void test_set_site_cookie() throws Exception { HttpServer server = httpServer(13423); server.get(eq(cookie("cookie"), "cookie-webmagic")).response("ok"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setUrl("http: Site site = Site.me().addCookie("cookie", "cookie-webmagic").setDomain("127.0.0.1"); Page page = httpClientDownloader.download(request, site.toTask()); assertThat(page.getRawText()).isEqualTo("ok"); } }); }
@Test public void test_download_when_task_is_null() throws Exception { HttpServer server = httpServer(13423); server.response("foo"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { final HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setUrl("http: Page page = httpClientDownloader.download(request, Site.me().toTask()); assertThat(page.getRawText()).isEqualTo("foo"); } }); }
@Test public void test_download_binary_content() throws Exception { HttpServer server = httpServer(13423); server.response("binary"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { final HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setBinaryContent(true); request.setUrl("http: Page page = httpClientDownloader.download(request, Site.me().toTask()); assertThat(page.getRawText()).isNull(); assertThat(page.getBytes()).isEqualTo("binary".getBytes()); } }); }
@Test public void test_download_set_charset() throws Exception { HttpServer server = httpServer(13423); server.response(header("Content-Type","text/html; charset=utf-8")).response("hello world!"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { final HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setUrl("http: Page page = httpClientDownloader.download(request, Site.me().toTask()); assertThat(page.getCharset()).isEqualTo("utf-8"); } }); }
@Test public void test_download_set_request_charset() throws Exception { HttpServer server = httpServer(13423); server.response("hello world!"); Runner.running(server, new Runnable() { @Override public void run() throws Exception { final HttpClientDownloader httpClientDownloader = new HttpClientDownloader(); Request request = new Request(); request.setCharset("utf-8"); request.setUrl("http: Page page = httpClientDownloader.download(request, Site.me().setCharset("gbk").toTask()); assertThat(page.getCharset()).isEqualTo("utf-8"); } }); }