@Test public void fetch() throws Exception { long start = System.currentTimeMillis(); FetchedData data1 = fetcher.fetch(indexPage); assertEquals("text/html", data1.getContentType()); assertNotNull(data1.getFetchedAt()); assertTrue(data1.getFetchedAt().getTime() >= start); assertTrue(data1.getFetchedAt().getTime() <= System.currentTimeMillis()); assertEquals(200, data1.getResponseCode()); try { fetcher.fetch(notFound); fail("Exception expected"); } catch (Exception e){ } }
@Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); @Override void init(JobContext context, String pluginId); String getUserAgent(); @Override Iterator<FetchedData> fetch(Iterator<Resource> resources); FetchedData fetch(Resource resource); @Override FetchedData apply(Resource resource); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); @Override void init(JobContext context, String pluginId); String getUserAgent(); @Override Iterator<FetchedData> fetch(Iterator<Resource> resources); FetchedData fetch(Resource resource); @Override FetchedData apply(Resource resource); static final Logger LOG; static final Integer CONNECT_TIMEOUT; static final Integer READ_TIMEOUT; static final Integer CONTENT_LIMIT; static final Integer DEFAULT_ERROR_CODE; static final String USER_AGENT; static final String TRUNCATED; }