@Test public void canFailLinkCheckWhenUrlIsNonExistentWithDefaultLimits() throws Exception { final SlaveLinkChecker slaveLinkChecker = new SlaveLinkChecker(); final HttpRetrieveResponse response = httpRetrieveResponseFactory.create(ResponseType.NO_STORAGE,null); final RetrieveUrl task = new RetrieveUrl(text1GitHubUrl+"-some-extra-nonsense", new ProcessingJobLimits(), DocumentReferenceTaskType.CHECK_LINK, "jobid-1", "referenceid-1", Collections.<String, String>emptyMap(), new ProcessingJobTaskDocumentReference(DocumentReferenceTaskType.CHECK_LINK, "source-reference-1", Collections.<ProcessingJobSubTask>emptyList()), null,new ReferenceOwner("unknown","unknwon","unknown")); slaveLinkChecker.downloadAndStoreInHttpRetrievResponse(response, task); assertEquals(RetrievingState.ERROR, response.getState()); assertEquals(404, response.getHttpResponseCode().longValue()); assertNotNull(response.getSourceIp()); assertTrue(response.getSocketConnectToDownloadStartDurationInMilliSecs() > 5); assertTrue(response.getCheckingDurationInMilliSecs() > 50); assertTrue(response.getRetrievalDurationInMilliSecs() > 50); }
public void downloadAndStoreInHttpRetrievResponse(final HttpRetrieveResponse httpRetrieveResponse, final RetrieveUrl task) { if ((task.getDocumentReferenceTask().getTaskType() != DocumentReferenceTaskType.CHECK_LINK)) { throw new IllegalArgumentException("The link checker can handle only link checking downloads. Cannot handle "+task.getDocumentReferenceTask().getTaskType()); } httpRetrieveResponse.setState(RetrievingState.ERROR); final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new AsyncHttpClientConfig.Builder() .setMaxRedirects(task.getLimits().getRetrievalMaxNrOfRedirects()) .setFollowRedirect(true) .setConnectTimeout(100000) .setAcceptAnyCertificate(true) .setMaxRequestRetry(3) .build()); httpRetrieveResponse.setState(RetrievingState.PROCESSING); httpRetrieveResponse.setRetrievalDurationInMilliSecs(0l); final long connectionSetupStartTimestamp = System.currentTimeMillis(); final ListenableFuture<Integer> downloadListener = asyncHttpClient.prepareGet(task.getUrl()).execute(new AsyncHandler<Integer>() { @Override public STATE onStatusReceived(HttpResponseStatus status) throws Exception { final long connectionSetupDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setSocketConnectToDownloadStartDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setCheckingDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setUrl(new URL(task.getUrl())); httpRetrieveResponse.setSourceIp(Slave.URL_RESOLVER.resolveIpOfUrl(task.getUrl())); if (connectionSetupDurationInMillis > task.getLimits().getRetrievalConnectionTimeoutInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } if (connectionSetupDurationInMillis > task.getLimits().getRetrievalTerminationThresholdReadPerSecondInBytes()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } httpRetrieveResponse.setHttpResponseCode(status.getStatusCode()); return STATE.CONTINUE; } @Override public STATE onHeadersReceived(HttpResponseHeaders downloadResponseHeaders) throws Exception { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setRetrievalDurationInMilliSecs(downloadDurationInMillis); for (final Map.Entry<String, List<String>> entry : downloadResponseHeaders.getHeaders()) { for (final String header : entry.getValue()) { httpRetrieveResponse.addHeader(entry.getKey(), header); } } if (httpRetrieveResponse.getHttpResponseCode() >= 400) { httpRetrieveResponse.setState(RetrievingState.ERROR); httpRetrieveResponse.setLog("A HTTP error received (code >= 400), the download did not initiate."); return STATE.ABORT; } httpRetrieveResponse.setState(RetrievingState.COMPLETED); return STATE.ABORT; } @Override public Integer onCompleted() throws Exception { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); return 0; } @Override public void onThrowable(Throwable e) { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; if (downloadDurationInMillis > task.getLimits().getRetrievalTerminationThresholdTimeLimitInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time it took to verify the link was too long (" + downloadDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); } if (httpRetrieveResponse.getState() == RetrievingState.COMPLETED) { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); } else { cleanup(httpRetrieveResponse, asyncHttpClient, e); } } @Override public STATE onBodyPartReceived(HttpResponseBodyPart httpResponseBodyPart) throws Exception { return STATE.ABORT; } }); try { Integer r = downloadListener.get(1, TimeUnit.DAYS ); LOG.debug(append(LogMarker.EUROPEANA_PROCESSING_JOB_ID, task.getJobId()),"Download finished with status: {}", r); } catch (Exception e) { cleanup(httpRetrieveResponse, asyncHttpClient, e); } finally { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); asyncHttpClient.close(); } }
SlaveLinkChecker { public void downloadAndStoreInHttpRetrievResponse(final HttpRetrieveResponse httpRetrieveResponse, final RetrieveUrl task) { if ((task.getDocumentReferenceTask().getTaskType() != DocumentReferenceTaskType.CHECK_LINK)) { throw new IllegalArgumentException("The link checker can handle only link checking downloads. Cannot handle "+task.getDocumentReferenceTask().getTaskType()); } httpRetrieveResponse.setState(RetrievingState.ERROR); final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new AsyncHttpClientConfig.Builder() .setMaxRedirects(task.getLimits().getRetrievalMaxNrOfRedirects()) .setFollowRedirect(true) .setConnectTimeout(100000) .setAcceptAnyCertificate(true) .setMaxRequestRetry(3) .build()); httpRetrieveResponse.setState(RetrievingState.PROCESSING); httpRetrieveResponse.setRetrievalDurationInMilliSecs(0l); final long connectionSetupStartTimestamp = System.currentTimeMillis(); final ListenableFuture<Integer> downloadListener = asyncHttpClient.prepareGet(task.getUrl()).execute(new AsyncHandler<Integer>() { @Override public STATE onStatusReceived(HttpResponseStatus status) throws Exception { final long connectionSetupDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setSocketConnectToDownloadStartDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setCheckingDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setUrl(new URL(task.getUrl())); httpRetrieveResponse.setSourceIp(Slave.URL_RESOLVER.resolveIpOfUrl(task.getUrl())); if (connectionSetupDurationInMillis > task.getLimits().getRetrievalConnectionTimeoutInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } if (connectionSetupDurationInMillis > task.getLimits().getRetrievalTerminationThresholdReadPerSecondInBytes()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } httpRetrieveResponse.setHttpResponseCode(status.getStatusCode()); return STATE.CONTINUE; } @Override public STATE onHeadersReceived(HttpResponseHeaders downloadResponseHeaders) throws Exception { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setRetrievalDurationInMilliSecs(downloadDurationInMillis); for (final Map.Entry<String, List<String>> entry : downloadResponseHeaders.getHeaders()) { for (final String header : entry.getValue()) { httpRetrieveResponse.addHeader(entry.getKey(), header); } } if (httpRetrieveResponse.getHttpResponseCode() >= 400) { httpRetrieveResponse.setState(RetrievingState.ERROR); httpRetrieveResponse.setLog("A HTTP error received (code >= 400), the download did not initiate."); return STATE.ABORT; } httpRetrieveResponse.setState(RetrievingState.COMPLETED); return STATE.ABORT; } @Override public Integer onCompleted() throws Exception { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); return 0; } @Override public void onThrowable(Throwable e) { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; if (downloadDurationInMillis > task.getLimits().getRetrievalTerminationThresholdTimeLimitInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time it took to verify the link was too long (" + downloadDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); } if (httpRetrieveResponse.getState() == RetrievingState.COMPLETED) { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); } else { cleanup(httpRetrieveResponse, asyncHttpClient, e); } } @Override public STATE onBodyPartReceived(HttpResponseBodyPart httpResponseBodyPart) throws Exception { return STATE.ABORT; } }); try { Integer r = downloadListener.get(1, TimeUnit.DAYS ); LOG.debug(append(LogMarker.EUROPEANA_PROCESSING_JOB_ID, task.getJobId()),"Download finished with status: {}", r); } catch (Exception e) { cleanup(httpRetrieveResponse, asyncHttpClient, e); } finally { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); asyncHttpClient.close(); } } }
SlaveLinkChecker { public void downloadAndStoreInHttpRetrievResponse(final HttpRetrieveResponse httpRetrieveResponse, final RetrieveUrl task) { if ((task.getDocumentReferenceTask().getTaskType() != DocumentReferenceTaskType.CHECK_LINK)) { throw new IllegalArgumentException("The link checker can handle only link checking downloads. Cannot handle "+task.getDocumentReferenceTask().getTaskType()); } httpRetrieveResponse.setState(RetrievingState.ERROR); final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new AsyncHttpClientConfig.Builder() .setMaxRedirects(task.getLimits().getRetrievalMaxNrOfRedirects()) .setFollowRedirect(true) .setConnectTimeout(100000) .setAcceptAnyCertificate(true) .setMaxRequestRetry(3) .build()); httpRetrieveResponse.setState(RetrievingState.PROCESSING); httpRetrieveResponse.setRetrievalDurationInMilliSecs(0l); final long connectionSetupStartTimestamp = System.currentTimeMillis(); final ListenableFuture<Integer> downloadListener = asyncHttpClient.prepareGet(task.getUrl()).execute(new AsyncHandler<Integer>() { @Override public STATE onStatusReceived(HttpResponseStatus status) throws Exception { final long connectionSetupDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setSocketConnectToDownloadStartDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setCheckingDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setUrl(new URL(task.getUrl())); httpRetrieveResponse.setSourceIp(Slave.URL_RESOLVER.resolveIpOfUrl(task.getUrl())); if (connectionSetupDurationInMillis > task.getLimits().getRetrievalConnectionTimeoutInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } if (connectionSetupDurationInMillis > task.getLimits().getRetrievalTerminationThresholdReadPerSecondInBytes()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } httpRetrieveResponse.setHttpResponseCode(status.getStatusCode()); return STATE.CONTINUE; } @Override public STATE onHeadersReceived(HttpResponseHeaders downloadResponseHeaders) throws Exception { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setRetrievalDurationInMilliSecs(downloadDurationInMillis); for (final Map.Entry<String, List<String>> entry : downloadResponseHeaders.getHeaders()) { for (final String header : entry.getValue()) { httpRetrieveResponse.addHeader(entry.getKey(), header); } } if (httpRetrieveResponse.getHttpResponseCode() >= 400) { httpRetrieveResponse.setState(RetrievingState.ERROR); httpRetrieveResponse.setLog("A HTTP error received (code >= 400), the download did not initiate."); return STATE.ABORT; } httpRetrieveResponse.setState(RetrievingState.COMPLETED); return STATE.ABORT; } @Override public Integer onCompleted() throws Exception { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); return 0; } @Override public void onThrowable(Throwable e) { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; if (downloadDurationInMillis > task.getLimits().getRetrievalTerminationThresholdTimeLimitInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time it took to verify the link was too long (" + downloadDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); } if (httpRetrieveResponse.getState() == RetrievingState.COMPLETED) { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); } else { cleanup(httpRetrieveResponse, asyncHttpClient, e); } } @Override public STATE onBodyPartReceived(HttpResponseBodyPart httpResponseBodyPart) throws Exception { return STATE.ABORT; } }); try { Integer r = downloadListener.get(1, TimeUnit.DAYS ); LOG.debug(append(LogMarker.EUROPEANA_PROCESSING_JOB_ID, task.getJobId()),"Download finished with status: {}", r); } catch (Exception e) { cleanup(httpRetrieveResponse, asyncHttpClient, e); } finally { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); asyncHttpClient.close(); } }  }
SlaveLinkChecker { public void downloadAndStoreInHttpRetrievResponse(final HttpRetrieveResponse httpRetrieveResponse, final RetrieveUrl task) { if ((task.getDocumentReferenceTask().getTaskType() != DocumentReferenceTaskType.CHECK_LINK)) { throw new IllegalArgumentException("The link checker can handle only link checking downloads. Cannot handle "+task.getDocumentReferenceTask().getTaskType()); } httpRetrieveResponse.setState(RetrievingState.ERROR); final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new AsyncHttpClientConfig.Builder() .setMaxRedirects(task.getLimits().getRetrievalMaxNrOfRedirects()) .setFollowRedirect(true) .setConnectTimeout(100000) .setAcceptAnyCertificate(true) .setMaxRequestRetry(3) .build()); httpRetrieveResponse.setState(RetrievingState.PROCESSING); httpRetrieveResponse.setRetrievalDurationInMilliSecs(0l); final long connectionSetupStartTimestamp = System.currentTimeMillis(); final ListenableFuture<Integer> downloadListener = asyncHttpClient.prepareGet(task.getUrl()).execute(new AsyncHandler<Integer>() { @Override public STATE onStatusReceived(HttpResponseStatus status) throws Exception { final long connectionSetupDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setSocketConnectToDownloadStartDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setCheckingDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setUrl(new URL(task.getUrl())); httpRetrieveResponse.setSourceIp(Slave.URL_RESOLVER.resolveIpOfUrl(task.getUrl())); if (connectionSetupDurationInMillis > task.getLimits().getRetrievalConnectionTimeoutInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } if (connectionSetupDurationInMillis > task.getLimits().getRetrievalTerminationThresholdReadPerSecondInBytes()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } httpRetrieveResponse.setHttpResponseCode(status.getStatusCode()); return STATE.CONTINUE; } @Override public STATE onHeadersReceived(HttpResponseHeaders downloadResponseHeaders) throws Exception { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setRetrievalDurationInMilliSecs(downloadDurationInMillis); for (final Map.Entry<String, List<String>> entry : downloadResponseHeaders.getHeaders()) { for (final String header : entry.getValue()) { httpRetrieveResponse.addHeader(entry.getKey(), header); } } if (httpRetrieveResponse.getHttpResponseCode() >= 400) { httpRetrieveResponse.setState(RetrievingState.ERROR); httpRetrieveResponse.setLog("A HTTP error received (code >= 400), the download did not initiate."); return STATE.ABORT; } httpRetrieveResponse.setState(RetrievingState.COMPLETED); return STATE.ABORT; } @Override public Integer onCompleted() throws Exception { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); return 0; } @Override public void onThrowable(Throwable e) { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; if (downloadDurationInMillis > task.getLimits().getRetrievalTerminationThresholdTimeLimitInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time it took to verify the link was too long (" + downloadDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); } if (httpRetrieveResponse.getState() == RetrievingState.COMPLETED) { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); } else { cleanup(httpRetrieveResponse, asyncHttpClient, e); } } @Override public STATE onBodyPartReceived(HttpResponseBodyPart httpResponseBodyPart) throws Exception { return STATE.ABORT; } }); try { Integer r = downloadListener.get(1, TimeUnit.DAYS ); LOG.debug(append(LogMarker.EUROPEANA_PROCESSING_JOB_ID, task.getJobId()),"Download finished with status: {}", r); } catch (Exception e) { cleanup(httpRetrieveResponse, asyncHttpClient, e); } finally { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); asyncHttpClient.close(); } }  void downloadAndStoreInHttpRetrievResponse(final HttpRetrieveResponse httpRetrieveResponse, final RetrieveUrl task); }
SlaveLinkChecker { public void downloadAndStoreInHttpRetrievResponse(final HttpRetrieveResponse httpRetrieveResponse, final RetrieveUrl task) { if ((task.getDocumentReferenceTask().getTaskType() != DocumentReferenceTaskType.CHECK_LINK)) { throw new IllegalArgumentException("The link checker can handle only link checking downloads. Cannot handle "+task.getDocumentReferenceTask().getTaskType()); } httpRetrieveResponse.setState(RetrievingState.ERROR); final AsyncHttpClient asyncHttpClient = new AsyncHttpClient(new AsyncHttpClientConfig.Builder() .setMaxRedirects(task.getLimits().getRetrievalMaxNrOfRedirects()) .setFollowRedirect(true) .setConnectTimeout(100000) .setAcceptAnyCertificate(true) .setMaxRequestRetry(3) .build()); httpRetrieveResponse.setState(RetrievingState.PROCESSING); httpRetrieveResponse.setRetrievalDurationInMilliSecs(0l); final long connectionSetupStartTimestamp = System.currentTimeMillis(); final ListenableFuture<Integer> downloadListener = asyncHttpClient.prepareGet(task.getUrl()).execute(new AsyncHandler<Integer>() { @Override public STATE onStatusReceived(HttpResponseStatus status) throws Exception { final long connectionSetupDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setSocketConnectToDownloadStartDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setCheckingDurationInMilliSecs(connectionSetupDurationInMillis); httpRetrieveResponse.setUrl(new URL(task.getUrl())); httpRetrieveResponse.setSourceIp(Slave.URL_RESOLVER.resolveIpOfUrl(task.getUrl())); if (connectionSetupDurationInMillis > task.getLimits().getRetrievalConnectionTimeoutInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } if (connectionSetupDurationInMillis > task.getLimits().getRetrievalTerminationThresholdReadPerSecondInBytes()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time to initiate the connection took too long (" + connectionSetupDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); return STATE.ABORT; } httpRetrieveResponse.setHttpResponseCode(status.getStatusCode()); return STATE.CONTINUE; } @Override public STATE onHeadersReceived(HttpResponseHeaders downloadResponseHeaders) throws Exception { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; httpRetrieveResponse.setRetrievalDurationInMilliSecs(downloadDurationInMillis); for (final Map.Entry<String, List<String>> entry : downloadResponseHeaders.getHeaders()) { for (final String header : entry.getValue()) { httpRetrieveResponse.addHeader(entry.getKey(), header); } } if (httpRetrieveResponse.getHttpResponseCode() >= 400) { httpRetrieveResponse.setState(RetrievingState.ERROR); httpRetrieveResponse.setLog("A HTTP error received (code >= 400), the download did not initiate."); return STATE.ABORT; } httpRetrieveResponse.setState(RetrievingState.COMPLETED); return STATE.ABORT; } @Override public Integer onCompleted() throws Exception { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); return 0; } @Override public void onThrowable(Throwable e) { final long downloadDurationInMillis = System.currentTimeMillis() - connectionSetupStartTimestamp; if (downloadDurationInMillis > task.getLimits().getRetrievalTerminationThresholdTimeLimitInMillis()) { httpRetrieveResponse.setState(RetrievingState.FINISHED_TIME_LIMIT); httpRetrieveResponse.setLog("The link could not be verified, as the time it took to verify the link was too long (" + downloadDurationInMillis + " ms longer than " + task.getLimits().getRetrievalConnectionTimeoutInMillis() + " ms)"); } if (httpRetrieveResponse.getState() == RetrievingState.COMPLETED) { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); } else { cleanup(httpRetrieveResponse, asyncHttpClient, e); } } @Override public STATE onBodyPartReceived(HttpResponseBodyPart httpResponseBodyPart) throws Exception { return STATE.ABORT; } }); try { Integer r = downloadListener.get(1, TimeUnit.DAYS ); LOG.debug(append(LogMarker.EUROPEANA_PROCESSING_JOB_ID, task.getJobId()),"Download finished with status: {}", r); } catch (Exception e) { cleanup(httpRetrieveResponse, asyncHttpClient, e); } finally { cleanup(httpRetrieveResponse, asyncHttpClient, httpRetrieveResponse.getException()); asyncHttpClient.close(); } }  void downloadAndStoreInHttpRetrievResponse(final HttpRetrieveResponse httpRetrieveResponse, final RetrieveUrl task);  }