StorageFileHelper { public void validateRegisteredS3Files(List<StorageFile> expectedStorageFiles, List<S3ObjectSummary> s3ObjectSummaries, String storageName, BusinessObjectDataKey businessObjectDataKey) { Set<String> actualS3FilePaths = new HashSet<>(getFilePathsFromS3ObjectSummaries(s3ObjectSummaries)); for (StorageFile expectedStorageFile : expectedStorageFiles) { if (!actualS3FilePaths.contains(expectedStorageFile.getFilePath())) { throw new ObjectNotFoundException( String.format("Registered file \"%s\" does not exist in \"%s\" storage.", expectedStorageFile.getFilePath(), storageName)); } } Set<String> expectedFilePaths = new HashSet<>(getFilePathsFromStorageFiles(expectedStorageFiles)); String businessObjectDataKeyAsJson = jsonHelper.objectToJson(businessObjectDataKey); for (S3ObjectSummary s3ObjectSummary : s3ObjectSummaries) { if (!expectedFilePaths.contains(s3ObjectSummary.getKey())) { if (s3ObjectSummary.getSize() == 0) { LOGGER.info("Ignoring unregistered zero byte S3 file. s3Key=\"{}\" storageName=\"{}\" businessObjectDataKey={}", s3ObjectSummary.getKey(), storageName, businessObjectDataKeyAsJson); } else { throw new IllegalStateException(String .format("Found unregistered non-empty S3 file \"%s\" in \"%s\" storage. Business object data {%s}", s3ObjectSummary.getKey(), storageName, businessObjectDataHelper.businessObjectDataKeyToString(businessObjectDataKey))); } } } }  List<StorageFile> createStorageFilesFromEntities(Collection<StorageFileEntity> storageFileEntities, String storageUnitDirectoryPath); List<StorageFile> createStorageFilesFromEntities(Collection<StorageFileEntity> storageFileEntities); List<StorageFile> createStorageFilesFromS3ObjectSummaries(List<S3ObjectSummary> s3ObjectSummaries); List<StorageFile> getAndValidateStorageFiles(StorageUnitEntity storageUnitEntity, String s3KeyPrefix, String storageName,
        BusinessObjectDataKey businessObjectDataKey); List<StorageFile> getAndValidateStorageFilesIfPresent(StorageUnitEntity storageUnitEntity, String s3KeyPrefix, String storageName,
        BusinessObjectDataKey businessObjectDataKey); List<String> getFilePathsFromS3ObjectSummaries(List<S3ObjectSummary> s3ObjectSummaries); List<String> getFilePathsFromStorageFiles(List<StorageFile> storageFiles); List<File> getFiles(List<StorageFile> storageFiles); Map<String, StorageFileEntity> getStorageFileEntitiesMap(Collection<StorageFileEntity> storageFileEntities); Map<String, StorageFile> getStorageFilesMapFromS3ObjectSummaries(List<S3ObjectSummary> s3ObjectSummaries); void validateCreateRequestStorageFiles(List<StorageFile> storageFiles); void validateDownloadedS3Files(String baseDirectory, String s3KeyPrefix, StorageUnit storageUnit); void validateDownloadedS3Files(String baseDirectory, String s3KeyPrefix, List<StorageFile> storageFiles); void validateRegisteredS3Files(List<StorageFile> expectedStorageFiles, List<S3ObjectSummary> s3ObjectSummaries, String storageName,
        BusinessObjectDataKey businessObjectDataKey); void validateStorageFile(StorageFile storageFile, String s3BucketName, Map<String, StorageFile> actualS3Keys, boolean validateFileSize); void validateStorageFileEntity(StorageFileEntity storageFileEntity, String s3BucketName, Map<String, StorageFile> actualS3Keys,
        boolean validateFileSize); void validateStorageFilePaths(Collection<String> storageFilePaths, String s3KeyPrefix, BusinessObjectDataKey businessObjectDataKey,
        String storageName); void validateStorageUnitS3Files(StorageUnit storageUnit, List<String> actualS3Files, String s3KeyPrefix);  }
@Test public void testValidateRegisteredS3Files() { List<StorageFile> testExpectedFiles = new ArrayList<>(); List<S3ObjectSummary> testActualFiles = new ArrayList<>(); for (String file : LOCAL_FILES) { String filePath = String.format("%s/%s", TEST_S3_KEY_PREFIX, file); testExpectedFiles.add(new StorageFile(filePath, FILE_SIZE, ROW_COUNT)); testActualFiles.add(createS3ObjectSummary(filePath, FILE_SIZE)); } storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION)); }
@Test public void testValidateRegisteredS3FilesActualFileNoExists() throws IOException { List<StorageFile> testExpectedFiles = Collections.singletonList(new StorageFile(TARGET_S3_KEY, FILE_SIZE, ROW_COUNT_1000)); List<S3ObjectSummary> testActualFiles = new ArrayList<>(); try { storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION)); fail("Should throw an ObjectNotFoundException when the registered S3 file does not exist."); } catch (ObjectNotFoundException e) { assertEquals(String.format("Registered file \"%s\" does not exist in \"%s\" storage.", TARGET_S3_KEY, STORAGE_NAME), e.getMessage()); } }
@Test public void testValidateRegisteredS3FilesUnexpectedZeroByteS3FileFound() throws IOException { List<StorageFile> testExpectedFiles = new ArrayList<>(); List<S3ObjectSummary> testActualFiles = Collections.singletonList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_0_BYTE)); BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION); storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey); }
@Test public void testValidateRegisteredS3FilesUnexpectedNonEmptyS3FileFound() throws IOException { List<StorageFile> testExpectedFiles = new ArrayList<>(); List<S3ObjectSummary> testActualFiles = Collections.singletonList(createS3ObjectSummary(TARGET_S3_KEY, FILE_SIZE_1_KB)); BusinessObjectDataKey businessObjectDataKey = new BusinessObjectDataKey(BDEF_NAMESPACE, BDEF_NAME, FORMAT_USAGE_CODE, FORMAT_FILE_TYPE_CODE, FORMAT_VERSION, PARTITION_VALUE, SUBPARTITION_VALUES, DATA_VERSION); try { storageFileHelper.validateRegisteredS3Files(testExpectedFiles, testActualFiles, STORAGE_NAME, businessObjectDataKey); fail(); } catch (IllegalStateException e) { assertEquals(String.format("Found unregistered non-empty S3 file \"%s\" in \"%s\" storage. Business object data {%s}", TARGET_S3_KEY, STORAGE_NAME, businessObjectDataServiceTestHelper.getExpectedBusinessObjectDataKeyAsString(businessObjectDataKey)), e.getMessage()); } }