@Test public void fetch1() throws Exception { Iterator<FetchedData> stream = fetcher.fetch(resources.iterator()); List<FetchedData> list = new ArrayList<>(); stream.forEachRemaining(list::add); assertEquals(list.size(), resources.size()); for (FetchedData data : list) { if (data.getResource().getUrl().contains("NOT_FOUND")) { assertEquals(404, data.getResponseCode()); } else { assertEquals(200, data.getResponseCode()); } } }
@Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); @Override void init(JobContext context, String pluginId); String getUserAgent(); @Override Iterator<FetchedData> fetch(Iterator<Resource> resources); FetchedData fetch(Resource resource); @Override FetchedData apply(Resource resource); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); @Override void init(JobContext context, String pluginId); String getUserAgent(); @Override Iterator<FetchedData> fetch(Iterator<Resource> resources); FetchedData fetch(Resource resource); @Override FetchedData apply(Resource resource); static final Logger LOG; static final Integer CONNECT_TIMEOUT; static final Integer READ_TIMEOUT; static final Integer CONTENT_LIMIT; static final Integer DEFAULT_ERROR_CODE; static final String USER_AGENT; static final String TRUNCATED; }