UploaderController extends DataBridgeController { @SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.") public void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params, Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs) throws InterruptedException, JAXBException, IOException, URISyntaxException, NoSuchAlgorithmException, KeyStoreException, KeyManagementException { boolean cleanUpS3KeyPrefixOnFailure = false; BusinessObjectDataKey businessObjectDataKey = null; try { UploaderInputManifestDto manifest = manifestReader.readJsonManifest(manifestPath); String storageName = getStorageNameFromManifest(manifest); manifest.setStorageName(storageName); List<File> sourceFiles = getValidatedLocalFiles(params.getLocalPath(), manifest.getManifestFiles()); List<File> duplicateFiles = findDuplicateFiles(sourceFiles); if (!duplicateFiles.isEmpty()) { throw new IllegalArgumentException( String.format("Manifest contains duplicate file names. Duplicates: [\"%s\"]", StringUtils.join(duplicateFiles, "\", \""))); } uploaderWebClient.setRegServerAccessParamsDto(regServerAccessParamsDto); checkLatestBusinessObjectDataVersion(manifest, force); BusinessObjectData businessObjectData = uploaderWebClient.preRegisterBusinessObjectData(manifest, storageName, createNewVersion); businessObjectDataKey = businessObjectDataHelper.getBusinessObjectDataKey(businessObjectData); Integer businessObjectDataVersion = businessObjectDataKey.getBusinessObjectDataVersion(); params.getAdditionalAwsCredentialsProviders().add(new AutoRefreshCredentialProvider() { @Override public AwsCredential getNewAwsCredential() throws Exception { return uploaderWebClient.getBusinessObjectDataUploadCredential(manifest, storageName, businessObjectDataVersion, null).getAwsCredential(); } }); String s3KeyPrefix = IterableUtils.get(businessObjectData.getStorageUnits(), 0).getStorageDirectory().getDirectoryPath(); Storage storage = uploaderWebClient.getStorage(storageName); String s3BucketName = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_BUCKET_NAME), storage, true); String kmsKeyId = storageHelper.getStorageAttributeValueByName(configurationHelper.getProperty(ConfigurationValue.S3_ATTRIBUTE_NAME_KMS_KEY_ID), storage, false); params.setKmsKeyId(kmsKeyId); params.setMaxThreads(adjustIntegerValue(params.getMaxThreads(), MIN_THREADS, MAX_THREADS)); params.setS3BucketName(s3BucketName); params.setS3KeyPrefix(s3KeyPrefix + "/"); params.setFiles(sourceFiles); if (s3Service.listDirectory(params).isEmpty()) { cleanUpS3KeyPrefixOnFailure = true; } else { throw new IllegalStateException(String .format("The destination S3 folder is not empty. S3 Bucket Name: \"%s\". S3 key prefix: \"%s\".", params.getS3BucketName(), params.getS3KeyPrefix())); } s3Service.uploadFileList(params); if (LOGGER.isInfoEnabled()) { logS3KeyPrefixContents(params); } addStorageFilesWithRetry(businessObjectDataKey, manifest, params, storage.getName(), maxRetryAttempts, retryDelaySecs); uploaderWebClient.updateBusinessObjectDataStatus(businessObjectDataKey, BusinessObjectDataStatusEntity.VALID); } catch (InterruptedException | JAXBException | IOException | URISyntaxException e) { if (cleanUpS3KeyPrefixOnFailure) { LOGGER.info(String .format("Rolling back the S3 data transfer by deleting keys/objects with prefix \"%s\" from bucket \"%s\".", params.getS3KeyPrefix(), params.getS3BucketName())); s3Service.deleteDirectoryIgnoreException(params); } if (businessObjectDataKey != null) { uploaderWebClient.updateBusinessObjectDataStatusIgnoreException(businessObjectDataKey, BusinessObjectDataStatusEntity.INVALID); } throw e; } }  @SuppressFBWarnings(value = "BC_UNCONFIRMED_CAST_OF_RETURN_VALUE", justification = "manifestReader.readJsonManifest will always return an UploaderInputManifestDto object.") void performUpload(RegServerAccessParamsDto regServerAccessParamsDto, File manifestPath, S3FileTransferRequestParamsDto params,
        Boolean createNewVersion, Boolean force, Integer maxRetryAttempts, Integer retryDelaySecs);  }
@Test public void testPerformUpload() throws Exception { uploadAndRegisterTestDataParents(uploaderWebClient); runUpload(UploaderController.MIN_THREADS); }
@Test public void testPerformUploadInvalidLocalDir() throws Exception { File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), getTestUploaderInputManifestDto()); Assert.assertTrue(manifestFile.isFile()); S3FileTransferRequestParamsDto s3FileTransferRequestParamsDto = getTestS3FileTransferRequestParamsDto(); s3FileTransferRequestParamsDto.setLocalPath(Paths.get(LOCAL_TEMP_PATH_INPUT.toString(), "I_DO_NOT_EXIST").toString()); RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true) .withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build(); try { uploaderController.performUpload(regServerAccessParamsDto, manifestFile, s3FileTransferRequestParamsDto, false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS); fail("Should throw an IllegalArgumentException when local directory does not exist."); } catch (IllegalArgumentException e) { assertTrue(e.getMessage().startsWith("Invalid local base directory")); } }
@Test(expected = IllegalArgumentException.class) public void testPerformUploadManifestContainsDuplicateFileNames() throws Exception { List<ManifestFile> duplicateTestDataFiles = new ArrayList<>(); duplicateTestDataFiles.addAll(testManifestFiles); duplicateTestDataFiles.add(testManifestFiles.get(0)); for (ManifestFile manifestFile : testManifestFiles) { createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB); } UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto(); uploaderInputManifestDto.setManifestFiles(duplicateTestDataFiles); File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto); Assert.assertTrue(manifestFile.isFile()); RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true) .withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build(); uploaderController.performUpload(regServerAccessParamsDto, manifestFile, getTestS3FileTransferRequestParamsDto(), false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS); }
@Test(expected = IllegalArgumentException.class) public void testPerformUploadManifestFileNameDoesNotMatchActualFileName() throws Exception { List<ManifestFile> declaredManifestFiles = getManifestFilesFromFileNames(Arrays.asList("test-data-1.txt", "test-data-2.txt"), FILE_SIZE_1_KB); List<ManifestFile> actualManifestFiles = getManifestFilesFromFileNames(Arrays.asList("test-data-1.txt", "TEST-DATA-2.TXT"), FILE_SIZE_1_KB); for (ManifestFile manifestFile : actualManifestFiles) { createLocalFile(LOCAL_TEMP_PATH_INPUT.toString(), manifestFile.getFileName(), FILE_SIZE_1_KB); } UploaderInputManifestDto uploaderInputManifestDto = getTestUploaderInputManifestDto(); uploaderInputManifestDto.setManifestFiles(declaredManifestFiles); File manifestFile = createManifestFile(LOCAL_TEMP_PATH_INPUT.toString(), uploaderInputManifestDto); Assert.assertTrue(manifestFile.isFile()); RegServerAccessParamsDto regServerAccessParamsDto = RegServerAccessParamsDto.builder().withRegServerHost(WEB_SERVICE_HOSTNAME).withRegServerPort(WEB_SERVICE_HTTPS_PORT).withUseSsl(true) .withUsername(WEB_SERVICE_HTTPS_USERNAME).withPassword(WEB_SERVICE_HTTPS_PASSWORD).build(); uploaderController.performUpload(regServerAccessParamsDto, manifestFile, getTestS3FileTransferRequestParamsDto(), false, false, TEST_RETRY_ATTEMPTS, TEST_RETRY_DELAY_SECS); }