@Test public void canRetryDownloadAndRecoverWhenServerError() throws IOException, StorageException { REQUESTED_GET_FAILURES.set(2); when(_mockPropertiesUtil.getRemoteMediaDownloadRetries()) .thenReturn(2); Path localPath = _tempFolder.newFolder().toPath().resolve("temp_downloaded_media"); Media media = mock(Media.class); when(media.getUri()) .thenReturn(EXPECTED_URI.toString()); when(media.getLocalPath()) .thenReturn(localPath); assertFalse(Files.exists(localPath)); _s3StorageBackend.downloadFromS3(media, getS3Properties()::get); assertTrue(Files.exists(localPath)); String sha; try (InputStream is = Files.newInputStream(localPath)) { sha = DigestUtils.sha256Hex(is); } assertEquals(EXPECTED_HASH, sha); assertEquals(3, GET_COUNT.get()); }
public void downloadFromS3(Media media, Function<String, String> combinedProperties) throws StorageException { try { AmazonS3 s3Client = getS3DownloadClient(media.getUri(), combinedProperties); String[] pathParts = splitBucketAndObjectKey(media.getUri()); String bucket = pathParts[0]; String objectKey = pathParts[1]; s3Client.getObject(new GetObjectRequest(bucket, objectKey), media.getLocalPath().toFile()); } catch (SdkClientException e) { throw new StorageException(String.format("Failed to download \"%s\" due to %s", media.getUri(), e), e); } }
S3StorageBackend implements StorageBackend { public void downloadFromS3(Media media, Function<String, String> combinedProperties) throws StorageException { try { AmazonS3 s3Client = getS3DownloadClient(media.getUri(), combinedProperties); String[] pathParts = splitBucketAndObjectKey(media.getUri()); String bucket = pathParts[0]; String objectKey = pathParts[1]; s3Client.getObject(new GetObjectRequest(bucket, objectKey), media.getLocalPath().toFile()); } catch (SdkClientException e) { throw new StorageException(String.format("Failed to download \"%s\" due to %s", media.getUri(), e), e); } } }
S3StorageBackend implements StorageBackend { public void downloadFromS3(Media media, Function<String, String> combinedProperties) throws StorageException { try { AmazonS3 s3Client = getS3DownloadClient(media.getUri(), combinedProperties); String[] pathParts = splitBucketAndObjectKey(media.getUri()); String bucket = pathParts[0]; String objectKey = pathParts[1]; s3Client.getObject(new GetObjectRequest(bucket, objectKey), media.getLocalPath().toFile()); } catch (SdkClientException e) { throw new StorageException(String.format("Failed to download \"%s\" due to %s", media.getUri(), e), e); } } @Inject  S3StorageBackend(PropertiesUtil propertiesUtil,
                            LocalStorageBackend localStorageBackend,
                            InProgressBatchJobsService inProgressBatchJobsService,
                            AggregateJobPropertiesUtil aggregateJobPropertiesUtil); }
S3StorageBackend implements StorageBackend { public void downloadFromS3(Media media, Function<String, String> combinedProperties) throws StorageException { try { AmazonS3 s3Client = getS3DownloadClient(media.getUri(), combinedProperties); String[] pathParts = splitBucketAndObjectKey(media.getUri()); String bucket = pathParts[0]; String objectKey = pathParts[1]; s3Client.getObject(new GetObjectRequest(bucket, objectKey), media.getLocalPath().toFile()); } catch (SdkClientException e) { throw new StorageException(String.format("Failed to download \"%s\" due to %s", media.getUri(), e), e); } } @Inject  S3StorageBackend(PropertiesUtil propertiesUtil,
                            LocalStorageBackend localStorageBackend,
                            InProgressBatchJobsService inProgressBatchJobsService,
                            AggregateJobPropertiesUtil aggregateJobPropertiesUtil); @Override boolean canStore(JsonOutputObject outputObject); @Override URI store(JsonOutputObject outputObject); @Override boolean canStore(ArtifactExtractionRequest request); @Override Table<Integer, Integer, URI> storeArtifacts(ArtifactExtractionRequest request); @Override boolean canStore(MarkupResult markupResult); @Override void store(MarkupResult markupResult); static void validateS3Properties(Function<String, String> properties); static boolean requiresS3MediaDownload(Function<String, String> properties); static boolean requiresS3ResultUpload(Function<String, String> properties); void downloadFromS3(Media media, Function<String, String> combinedProperties); S3Object getFromS3(String uri, Function<String, String> properties); }
S3StorageBackend implements StorageBackend { public void downloadFromS3(Media media, Function<String, String> combinedProperties) throws StorageException { try { AmazonS3 s3Client = getS3DownloadClient(media.getUri(), combinedProperties); String[] pathParts = splitBucketAndObjectKey(media.getUri()); String bucket = pathParts[0]; String objectKey = pathParts[1]; s3Client.getObject(new GetObjectRequest(bucket, objectKey), media.getLocalPath().toFile()); } catch (SdkClientException e) { throw new StorageException(String.format("Failed to download \"%s\" due to %s", media.getUri(), e), e); } } @Inject  S3StorageBackend(PropertiesUtil propertiesUtil,
                            LocalStorageBackend localStorageBackend,
                            InProgressBatchJobsService inProgressBatchJobsService,
                            AggregateJobPropertiesUtil aggregateJobPropertiesUtil); @Override boolean canStore(JsonOutputObject outputObject); @Override URI store(JsonOutputObject outputObject); @Override boolean canStore(ArtifactExtractionRequest request); @Override Table<Integer, Integer, URI> storeArtifacts(ArtifactExtractionRequest request); @Override boolean canStore(MarkupResult markupResult); @Override void store(MarkupResult markupResult); static void validateS3Properties(Function<String, String> properties); static boolean requiresS3MediaDownload(Function<String, String> properties); static boolean requiresS3ResultUpload(Function<String, String> properties); void downloadFromS3(Media media, Function<String, String> combinedProperties); S3Object getFromS3(String uri, Function<String, String> properties);  }