RemoteAttachmentDownloader { public static int retrieveRemoteAttachments(Supplier<HttpClient> httpClient, String dbAttachments, Duration downloadTimeout) throws MalformedURLException { AttachmentConnector attachmentConnector = new AttachmentConnector(httpClient, dbAttachments, downloadTimeout); AttachmentRepository attachmentRepository = new AttachmentRepository(new DatabaseConnector(httpClient, dbAttachments)); List<AttachmentContent> remoteAttachments = attachmentRepository.getOnlyRemoteAttachments(); log.info(format("we have %d remote attachments to retrieve", remoteAttachments.size())); int count = 0; for (AttachmentContent attachmentContent : remoteAttachments) { if (!attachmentContent.isOnlyRemote()) { log.info(format("skipping attachment (%s), which should already be available", attachmentContent.getId())); continue; } String attachmentContentId = attachmentContent.getId(); log.info(format("retrieving attachment (%s) {filename=%s}", attachmentContentId, attachmentContent.getFilename())); log.debug("url is " + attachmentContent.getRemoteUrl()); InputStream content = null; try { content = attachmentConnector.unsafeGetAttachmentStream(attachmentContent); if (content == null) { log.error("null content retrieving attachment " + attachmentContentId); continue; } try { long length = length(content); log.info(format("retrieved attachment (%s), it was %d bytes long", attachmentContentId, length)); count++; } catch (IOException e) { log.error("attachment was downloaded but somehow not available in database " + attachmentContentId, e); } } catch (SW360Exception e) { log.error("cannot retrieve attachment " + attachmentContentId, e); } finally { closeQuietly(content, log); } } return count; }  static void main(String[] args); static int retrieveRemoteAttachments(Supplier<HttpClient> httpClient, String dbAttachments, Duration downloadTimeout);  }
@Test public void testIntegration() throws Exception { AttachmentContent attachmentContent = saveRemoteAttachment(url); assertThat(retrieveRemoteAttachments(DatabaseSettings.getConfiguredHttpClient(), dbName, downloadTimeout), is(1)); assertThat(attachmentConnector.getAttachmentStream(attachmentContent, dummyUser, new Project() .setVisbility(Visibility.ME_AND_MODERATORS) .setCreatedBy(dummyUser.getEmail()) .setAttachments(Collections.singleton(new Attachment().setAttachmentContentId(attachmentContent.getId())))), hasLength(greaterThan(0l))); assertThat(retrieveRemoteAttachments(DatabaseSettings.getConfiguredHttpClient(), dbName, downloadTimeout), is(0)); }
@Test public void testWithBlackHole() throws Exception { assumeThat(getAvailableNetworkInterface(), isAvailable()); saveRemoteAttachment("http: assertThat(repository.getOnlyRemoteAttachments(), hasSize(1)); ExecutorService executor = newSingleThreadExecutor(); try { Future<Integer> future = executor.submit(new Callable<Integer>() { @Override public Integer call() throws Exception { return retrieveRemoteAttachments(DatabaseSettings.getConfiguredHttpClient(), dbName, downloadTimeout); } }); try { Integer downloadedSuccessfully = future.get(1, TimeUnit.MINUTES); assertThat(downloadedSuccessfully, is(0)); } catch (TimeoutException e) { fail("retriever got stuck on a black hole"); throw e; } } finally { executor.shutdown(); executor.awaitTermination(1, TimeUnit.MINUTES); } assertThat(repository.getOnlyRemoteAttachments(), hasSize(1)); }
@Test public void testWithBrokenURL() throws Exception { AttachmentContent attachmentContent = saveRemoteAttachment(TestUtils.BLACK_HOLE_ADDRESS); AttachmentContent attachmentGood = saveRemoteAttachment(url); assertThat(repository.getOnlyRemoteAttachments(), hasSize(2)); assertThat(retrieveRemoteAttachments(DatabaseSettings.getConfiguredHttpClient(), dbName, downloadTimeout), is(1)); assertThat(repository.getOnlyRemoteAttachments(), hasSize(1)); assertThat(attachmentConnector.getAttachmentStream(attachmentGood, dummyUser, new Project() .setVisbility(Visibility.ME_AND_MODERATORS) .setCreatedBy(dummyUser.getEmail()) .setAttachments(Collections.singleton(new Attachment().setAttachmentContentId(attachmentGood.getId())))), hasLength(greaterThan(0l))); assertThat(repository.getOnlyRemoteAttachments(), hasSize(1)); assertThat(retrieveRemoteAttachments(DatabaseSettings.getConfiguredHttpClient(), dbName, downloadTimeout), is(0)); assertThat(repository.getOnlyRemoteAttachments(), hasSize(1)); try { assertThat(attachmentConnector.getAttachmentStream(attachmentContent, dummyUser, new Project() .setVisbility(Visibility.ME_AND_MODERATORS) .setCreatedBy(dummyUser.getEmail()) .setAttachments(Collections.singleton(new Attachment().setAttachmentContentId(attachmentContent.getId())))), hasLength(greaterThan(0l))); fail("expected exception not thrown"); } catch (SW360Exception e) { assertThat(e.getWhy(), containsString(attachmentContent.getId())); } }