DownloaderController extends DataBridgeController { @SuppressFBWarnings(value = {"BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.") public void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto) throws InterruptedException, JAXBException, IOException, URISyntaxException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { boolean cleanUpTargetLocalDirectoryOnFailure = false; File targetLocalDirectory = null; try { DownloaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath); String storageName = getStorageNameFromManifest(manifest); downloaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto); BusinessObjectData businessObjectData = downloaderWebClient.getBusinessObjectData(manifest); manifest.setBusinessObjectDataVersion(String.valueOf(businessObjectData.getVersion())); manifest.setBusinessObjectFormatVersion(String.valueOf(businessObjectData.getBusinessObjectFormatVersion())); s3FileTransferRequestParamsDto.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() { @Override public AwsCredential getNewAwsCredential() throws Exception { return downloaderWebClient.getStorageUnitDownloadCredential(manifest, storageName).getAwsCredential(); } }); StorageUnit storageUnit = businessObjectDataHelper.getStorageUnitByStorageName(businessObjectData, storageName); S3KeyPrefixInformation s3KeyPrefixInformation = downloaderWebClient.getS3KeyPrefix(businessObjectData); targetLocalDirectory = Paths.get(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix()).toFile(); if (!targetLocalDirectory.isDirectory()) { if (!targetLocalDirectory.mkdirs()) { throw new IllegalArgumentException(String.format("Failed to create target local directory \"%s\".", targetLocalDirectory.getPath())); } } else { if (targetLocalDirectory.list().length > 0) { throw new IllegalArgumentException(String.format("The target local directory \"%s\" is not empty.", targetLocalDirectory.getPath())); } } Storage storage = downloaderWebClient.getStorage(storageName); String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true); s3FileTransferRequestParamsDto.setS3BucketName(s3BucketName); s3FileTransferRequestParamsDto.setS3KeyPrefix(s3KeyPrefixInformation.getS3KeyPrefix() + "/"); List<String> actualS3Files = storageFileHelper.getFilePathsFromS3ObjectSummaries(s3Service.listDirectory(s3FileTransferRequestParamsDto, true)); storageFileHelper.validateStorageUnitS3Files(storageUnit, actualS3Files, s3KeyPrefixInformation.getS3KeyPrefix()); s3FileTransferRequestParamsDto.setMaxThreads(adjustIntegerValue(s3FileTransferRequestParamsDto.getMaxThreads(), MIN_THREADS, MAX_THREADS)); s3FileTransferRequestParamsDto.setRecursive(true); cleanUpTargetLocalDirectoryOnFailure = true; s3Service.downloadDirectory(s3FileTransferRequestParamsDto); storageFileHelper.validateDownloadedS3Files(s3FileTransferRequestParamsDto.getLocalPath(), s3KeyPrefixInformation.getS3KeyPrefix(), storageUnit); if (LOGGER.isInfoEnabled()) { logLocalDirectoryContents(targetLocalDirectory); } DownloaderOutputManifestDto downloaderOutputManifestDto = createDownloaderOutputManifestDto(businessObjectData, storageUnit, s3KeyPrefixInformation.getS3KeyPrefix()); manifestWriter.writeJsonManifest(targetLocalDirectory, OUTPUT_MANIFEST_FILE_NAME, downloaderOutputManifestDto); } catch (InterruptedException | JAXBException | IOException | URISyntaxException e) { if (cleanUpTargetLocalDirectoryOnFailure) { LOGGER.info(String.format("Rolling back the S3 data transfer by cleaning up \"%s\" target local directory.", targetLocalDirectory)); HerdFileUtils.cleanDirectoryIgnoreException(targetLocalDirectory); } throw e; } }  @SuppressFBWarnings(value = {"BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", "NP_NULL_ON_SOME_PATH_FROM_RETURN_VALUE"}, justification = "manifestReader.readJsonManifest will always return an DownloaderInputManifestDto object. targetLocalDirectory.list().length will not" + " return a NullPointerException.") void performDownload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath,
        S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto);  }
@Test public void testPerformDownload() throws Exception { runDownload(); }
@Test public void testPerformDownloadZeroByteDirectoryMarkersPresent() throws Exception { uploadAndRegisterTestDataParents(downloaderWebClient); uploadAndRegisterTestData(S3_TEST_PATH_V0, testManifestFiles, S3_DIRECTORY_MARKERS); File downloaderInputManifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), getTestDownloaderInputManifestDto()); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setLocalPath(LOCAL_TEMP_PATH_OUTPUT.toString()); s3FileTransferRequestParamsDto.setMaxThreads(DownloaderController.MIN_THREADS); RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true) .withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).withTrustSelfSignedCertificate(true) .withDisableHostnameVerification(true).build(); downloaderController.performDownload(regServerAccessParamsDto, downloaderInputManifestFile, s3FileTransferRequestParamsDto); }
@Test public void testPerformDownloadAssertCleanTargetDirectoryWhenError() throws Exception { DownloaderWebClient mockDownloaderWebClient = mock(DownloaderWebClient.class); DownloaderWebClient originalDownloaderWebClient = (DownloaderWebClient) ReflectionTestUtils.getField(downloaderController, "downloaderWebClient"); ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", mockDownloaderWebClient); DownloaderManifestReader mockDownloaderManifestReader = mock(DownloaderManifestReader.class); DownloaderManifestReader originalDownloaderManifestReader = (DownloaderManifestReader) ReflectionTestUtils.getField(downloaderController, "manifestReader"); ReflectionTestUtils.setField(downloaderController, "manifestReader", mockDownloaderManifestReader); BusinessObjectDataHelper mockBusinessObjectDataHelper = mock(BusinessObjectDataHelper.class); BusinessObjectDataHelper originalBusinessObjectDataHelper = (BusinessObjectDataHelper) ReflectionTestUtils.getField(downloaderController, "businessObjectDataHelper"); ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", mockBusinessObjectDataHelper); S3Service mockS3Service = mock(S3Service.class); S3Service originalS3Service = (S3Service) ReflectionTestUtils.getField(downloaderController, "s3Service"); ReflectionTestUtils.setField(downloaderController, "s3Service", mockS3Service); StorageFileHelper mockStorageFileHelper = mock(StorageFileHelper.class); StorageFileHelper originalStorageFileHelper = (StorageFileHelper) ReflectionTestUtils.getField(downloaderController, "storageFileHelper"); ReflectionTestUtils.setField(downloaderController, "storageFileHelper", mockStorageFileHelper); StorageHelper mockStorageHelper = mock(StorageHelper.class); StorageHelper originalStorageHelper = (StorageHelper) ReflectionTestUtils.getField(downloaderController, "storageHelper"); ReflectionTestUtils.setField(downloaderController, "storageHelper", mockStorageHelper); Path localPath = Files.createTempDirectory(null); try { String s3KeyPrefix = "s3KeyPrefix"; String storageName = "storageName"; IOException expectedException = new IOException(); Path targetDirectoryPath = localPath.resolve(s3KeyPrefix); DownloaderInputManifestDto downloaderInputManifestDto = new DownloaderInputManifestDto(); BusinessObjectData businessObjectData = new BusinessObjectData(); StorageUnit storageUnit = new StorageUnit(new Storage(storageName, null, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null); S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation(); s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix); when(mockDownloaderManifestReader.readJsonManifest(any())).thenReturn(downloaderInputManifestDto); when(mockDownloaderWebClient.getBusinessObjectData(any())).thenReturn(businessObjectData); when(mockBusinessObjectDataHelper.getStorageUnitByStorageName(any(), any())).thenReturn(storageUnit); when(mockDownloaderWebClient.getS3KeyPrefix(any())).thenReturn(s3KeyPrefixInformation); when(mockS3Service.downloadDirectory(any())).then(new Answer<S3FileTransferResultsDto>() { @Override public S3FileTransferResultsDto answer(InvocationOnMock invocation) throws Throwable { Files.createFile(targetDirectoryPath.resolve("file")); throw expectedException; } }); RegServerAccessParamsDto regServerAccessParamsDto = null; File manifestPath = null; S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setLocalPath(localPath.toString()); s3FileTransferRequestParamsDto.setMaxThreads(1); try { downloaderController.performDownload(regServerAccessParamsDto, manifestPath, s3FileTransferRequestParamsDto); fail(); } catch (Exception e) { assertEquals(expectedException, e); assertEquals(0, targetDirectoryPath.toFile().list().length); } } finally { ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", originalDownloaderWebClient); ReflectionTestUtils.setField(downloaderController, "manifestReader", originalDownloaderManifestReader); ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", originalBusinessObjectDataHelper); ReflectionTestUtils.setField(downloaderController, "s3Service", originalS3Service); ReflectionTestUtils.setField(downloaderController, "storageFileHelper", originalStorageFileHelper); ReflectionTestUtils.setField(downloaderController, "storageHelper", originalStorageHelper); FileUtils.deleteDirectory(localPath.toFile()); } }
@Test public void testPerformDownloadAssertCredentialsRetrieved() throws Exception { DownloaderWebClient mockDownloaderWebClient = mock(DownloaderWebClient.class); DownloaderWebClient originalDownloaderWebClient = (DownloaderWebClient) ReflectionTestUtils.getField(downloaderController, "downloaderWebClient"); ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", mockDownloaderWebClient); DownloaderManifestReader mockDownloaderManifestReader = mock(DownloaderManifestReader.class); DownloaderManifestReader originalDownloaderManifestReader = (DownloaderManifestReader) ReflectionTestUtils.getField(downloaderController, "manifestReader"); ReflectionTestUtils.setField(downloaderController, "manifestReader", mockDownloaderManifestReader); BusinessObjectDataHelper mockBusinessObjectDataHelper = mock(BusinessObjectDataHelper.class); BusinessObjectDataHelper originalBusinessObjectDataHelper = (BusinessObjectDataHelper) ReflectionTestUtils.getField(downloaderController, "businessObjectDataHelper"); ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", mockBusinessObjectDataHelper); S3Service mockS3Service = mock(S3Service.class); S3Service originalS3Service = (S3Service) ReflectionTestUtils.getField(downloaderController, "s3Service"); ReflectionTestUtils.setField(downloaderController, "s3Service", mockS3Service); StorageFileHelper mockStorageFileHelper = mock(StorageFileHelper.class); StorageFileHelper originalStorageFileHelper = (StorageFileHelper) ReflectionTestUtils.getField(downloaderController, "storageFileHelper"); ReflectionTestUtils.setField(downloaderController, "storageFileHelper", mockStorageFileHelper); StorageHelper mockStorageHelper = mock(StorageHelper.class); StorageHelper originalStorageHelper = (StorageHelper) ReflectionTestUtils.getField(downloaderController, "storageHelper"); ReflectionTestUtils.setField(downloaderController, "storageHelper", mockStorageHelper); Path localPath = Files.createTempDirectory(null); try { String s3KeyPrefix = "s3KeyPrefix"; String storageName = "storageName"; DownloaderInputManifestDto downloaderInputManifestDto = new DownloaderInputManifestDto(); downloaderInputManifestDto.setStorageName(storageName); BusinessObjectData businessObjectData = new BusinessObjectData(); StorageUnit storageUnit = new StorageUnit(new Storage(storageName, null, null), null, null, StorageUnitStatusEntity.ENABLED, null, null, null); S3KeyPrefixInformation s3KeyPrefixInformation = new S3KeyPrefixInformation(); s3KeyPrefixInformation.setS3KeyPrefix(s3KeyPrefix); when(mockDownloaderManifestReader.readJsonManifest(any())).thenReturn(downloaderInputManifestDto); when(mockDownloaderWebClient.getBusinessObjectData(any())).thenReturn(businessObjectData); when(mockBusinessObjectDataHelper.getStorageUnitByStorageName(any(), any())).thenReturn(storageUnit); when(mockDownloaderWebClient.getS3KeyPrefix(any())).thenReturn(s3KeyPrefixInformation); when(mockDownloaderWebClient.getStorageUnitDownloadCredential(any(), any())).thenReturn(new StorageUnitDownloadCredential( new AwsCredential("awsAccessKey", "awsSecretKey", "awsSessionToken", DatatypeFactory.newInstance().newXMLGregorianCalendar()))); when(mockS3Service.downloadDirectory(any())).then(new Answer<S3FileTransferResultsDto>() { @Override public S3FileTransferResultsDto answer(InvocationOnMock invocation) throws Throwable { S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = invocation.getArgument(0); List<HerdAWSCredentialsProvider> additionalAwsCredentialsProviders = s3FileTransferRequestParamsDto.getAdditionalAwsCredentialsProviders(); for (HerdAWSCredentialsProvider herdAWSCredentialsProvider : additionalAwsCredentialsProviders) { herdAWSCredentialsProvider.getAwsCredential(); } return null; } }); RegServerAccessParamsDto regServerAccessParamsDto = null; File manifestPath = null; S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = new S3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setLocalPath(localPath.toString()); s3FileTransferRequestParamsDto.setMaxThreads(1); downloaderController.performDownload(regServerAccessParamsDto, manifestPath, s3FileTransferRequestParamsDto); verify(mockDownloaderWebClient).getStorageUnitDownloadCredential(downloaderInputManifestDto, storageName); } finally { ReflectionTestUtils.setField(downloaderController, "downloaderWebClient", originalDownloaderWebClient); ReflectionTestUtils.setField(downloaderController, "manifestReader", originalDownloaderManifestReader); ReflectionTestUtils.setField(downloaderController, "businessObjectDataHelper", originalBusinessObjectDataHelper); ReflectionTestUtils.setField(downloaderController, "s3Service", originalS3Service); ReflectionTestUtils.setField(downloaderController, "storageFileHelper", originalStorageFileHelper); ReflectionTestUtils.setField(downloaderController, "storageHelper", originalStorageHelper); FileUtils.deleteDirectory(localPath.toFile()); } }