@Test public void testHeaders() throws Exception { String url = "http: Resource page = new Resource(url, "localhost", job); FetchedData fetchedData = fetcher.fetch(page); boolean headerFound = false; boolean userAgentFound = false; String[] lines = new String(fetchedData.getContent()).split("\n"); for (String line : lines) { String[] split = line.trim().split(":"); if (split[0].equals("Custom-Header")){ headerFound = split[1].trim().equals("Custom Header Value"); } if (split[0].equals("User-Agent")){ userAgentFound = split[1].trim().contains("Sparkler"); } } assertTrue(headerFound); assertTrue(userAgentFound); }
@Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); @Override void init(JobContext context, String pluginId); String getUserAgent(); @Override Iterator<FetchedData> fetch(Iterator<Resource> resources); FetchedData fetch(Resource resource); @Override FetchedData apply(Resource resource); }
FetcherDefault extends AbstractExtensionPoint implements Fetcher, Function<Resource, FetchedData> { @Override public Iterator<FetchedData> fetch(Iterator<Resource> resources) throws Exception { return new StreamTransformer<>(resources, this); } FetcherDefault(); @Override void init(JobContext context, String pluginId); String getUserAgent(); @Override Iterator<FetchedData> fetch(Iterator<Resource> resources); FetchedData fetch(Resource resource); @Override FetchedData apply(Resource resource); static final Logger LOG; static final Integer CONNECT_TIMEOUT; static final Integer READ_TIMEOUT; static final Integer CONTENT_LIMIT; static final Integer DEFAULT_ERROR_CODE; static final String USER_AGENT; static final String TRUNCATED; }