FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); @Override void init(JobContext context, String pluginId); String getUserAgent(); @Override Iterator<FetchedData> fetch(Iterator<Resource> resources); FetchedData fetch(Resource resource); @Override FetchedData apply(Resource resource); static final Logger LOG; static final Integer CONNECT_TIMEOUT; static final Integer READ_TIMEOUT; static final Integer CONTENT_LIMIT; static final Integer DEFAULT_ERROR_CODE; static final String USER_AGENT; static final String TRUNCATED; }
@Test public void fetch() throws Exception { long start = System.currentTimeMillis(); FetchedData data1 = fetcher.fetch(indexPage); assertEquals("text/html", data1.getContentType()); assertNotNull(data1.getFetchedAt()); assertTrue(data1.getFetchedAt().getTime() >= start); assertTrue(data1.getFetchedAt().getTime() <= System.currentTimeMillis()); assertEquals(200, data1.getResponseCode()); try { fetcher.fetch(notFound); fail("Exception expected"); } catch (Exception e){ } }
@Test public void fetch1() throws Exception { Iterator<FetchedData> stream = fetcher.fetch(resources.iterator()); List<FetchedData> list = new ArrayList<>(); stream.forEachRemaining(list::add); assertEquals(list.size(), resources.size()); for (FetchedData data : list) { if (data.getResource().getUrl().contains("NOT_FOUND")) { assertEquals(404, data.getResponseCode()); } else { assertEquals(200, data.getResponseCode()); } } }
@Test public void testReadTimeout() throws Exception { String url = "http: url += FetcherDefault.READ_TIMEOUT + 1000; Resource timeoutPage = new Resource(url, "localhost", job); try { fetcher.fetch(timeoutPage); fail("Exception expected"); } catch (SocketTimeoutException e){ } catch (Exception e){ fail("Socket Exception expected, but found:" + e.getClass().getName()); } }
@Test public void testHeaders() throws Exception { String url = "http: Resource page = new Resource(url, "localhost", job); FetchedData fetchedData = fetcher.fetch(page); boolean headerFound = false; boolean userAgentFound = false; String[] lines = new String(fetchedData.getContent()).split("\n"); for (String line : lines) { String[] split = line.trim().split(":"); if (split[0].equals("Custom-Header")){ headerFound = split[1].trim().equals("Custom Header Value"); } if (split[0].equals("User-Agent")){ userAgentFound = split[1].trim().contains("Sparkler"); } } assertTrue(headerFound); assertTrue(userAgentFound); }