@Test void reportBytesDeletedWhenDirectoryDeletionPartiallyFails() { AmazonS3 mockAmazonS3 = mock(AmazonS3.class); S3Client mockS3Client = new S3Client(mockAmazonS3, false); mockOneOutOfTwoObjectsDeleted(mockAmazonS3); s3PathCleaner = new S3PathCleaner(mockS3Client, s3SentinelFilesCleaner, bytesDeletedReporter); assertThatExceptionOfType(BeekeeperException.class) .isThrownBy(() -> s3PathCleaner.cleanupPath(housekeepingPath)) .withMessage(format("Not all files could be deleted at path \"%s/%s\"; deleted 1/2 objects. " + "Objects not deleted: 'table/id1/partition_1/file2'.", bucket, keyRootAsDirectory)); verify(bytesDeletedReporter).reportTaggable(100L, housekeepingPath, FileSystemType.S3); }
@Override @TimedTaggable("s3-paths-deleted") public void cleanupPath(HousekeepingEntity housekeepingEntity) { S3SchemeURI s3SchemeURI = new S3SchemeURI(housekeepingEntity.getPath()); String key = s3SchemeURI.getKey(); String bucket = s3SchemeURI.getBucket(); S3BytesDeletedCalculator bytesDeletedCalculator = new S3BytesDeletedCalculator(s3Client); boolean isFile = s3Client.doesObjectExist(bucket, key); try { if (isFile) { deleteFile(bucket, key, bytesDeletedCalculator); } else { deleteFilesInDirectory(bucket, key, bytesDeletedCalculator); deleteSentinelFiles(s3SchemeURI, key, bucket, housekeepingEntity.getTableName()); } } finally { long bytesDeleted = bytesDeletedCalculator.getBytesDeleted(); if (bytesDeleted > 0) { bytesDeletedReporter.reportTaggable(bytesDeleted, housekeepingEntity, FileSystemType.S3); } } }
S3PathCleaner implements PathCleaner { @Override @TimedTaggable("s3-paths-deleted") public void cleanupPath(HousekeepingEntity housekeepingEntity) { S3SchemeURI s3SchemeURI = new S3SchemeURI(housekeepingEntity.getPath()); String key = s3SchemeURI.getKey(); String bucket = s3SchemeURI.getBucket(); S3BytesDeletedCalculator bytesDeletedCalculator = new S3BytesDeletedCalculator(s3Client); boolean isFile = s3Client.doesObjectExist(bucket, key); try { if (isFile) { deleteFile(bucket, key, bytesDeletedCalculator); } else { deleteFilesInDirectory(bucket, key, bytesDeletedCalculator); deleteSentinelFiles(s3SchemeURI, key, bucket, housekeepingEntity.getTableName()); } } finally { long bytesDeleted = bytesDeletedCalculator.getBytesDeleted(); if (bytesDeleted > 0) { bytesDeletedReporter.reportTaggable(bytesDeleted, housekeepingEntity, FileSystemType.S3); } } } }
S3PathCleaner implements PathCleaner { @Override @TimedTaggable("s3-paths-deleted") public void cleanupPath(HousekeepingEntity housekeepingEntity) { S3SchemeURI s3SchemeURI = new S3SchemeURI(housekeepingEntity.getPath()); String key = s3SchemeURI.getKey(); String bucket = s3SchemeURI.getBucket(); S3BytesDeletedCalculator bytesDeletedCalculator = new S3BytesDeletedCalculator(s3Client); boolean isFile = s3Client.doesObjectExist(bucket, key); try { if (isFile) { deleteFile(bucket, key, bytesDeletedCalculator); } else { deleteFilesInDirectory(bucket, key, bytesDeletedCalculator); deleteSentinelFiles(s3SchemeURI, key, bucket, housekeepingEntity.getTableName()); } } finally { long bytesDeleted = bytesDeletedCalculator.getBytesDeleted(); if (bytesDeleted > 0) { bytesDeletedReporter.reportTaggable(bytesDeleted, housekeepingEntity, FileSystemType.S3); } } } S3PathCleaner(S3Client s3Client, SentinelFilesCleaner sentinelFilesCleaner,
      BytesDeletedReporter bytesDeletedReporter); }
S3PathCleaner implements PathCleaner { @Override @TimedTaggable("s3-paths-deleted") public void cleanupPath(HousekeepingEntity housekeepingEntity) { S3SchemeURI s3SchemeURI = new S3SchemeURI(housekeepingEntity.getPath()); String key = s3SchemeURI.getKey(); String bucket = s3SchemeURI.getBucket(); S3BytesDeletedCalculator bytesDeletedCalculator = new S3BytesDeletedCalculator(s3Client); boolean isFile = s3Client.doesObjectExist(bucket, key); try { if (isFile) { deleteFile(bucket, key, bytesDeletedCalculator); } else { deleteFilesInDirectory(bucket, key, bytesDeletedCalculator); deleteSentinelFiles(s3SchemeURI, key, bucket, housekeepingEntity.getTableName()); } } finally { long bytesDeleted = bytesDeletedCalculator.getBytesDeleted(); if (bytesDeleted > 0) { bytesDeletedReporter.reportTaggable(bytesDeleted, housekeepingEntity, FileSystemType.S3); } } } S3PathCleaner(S3Client s3Client, SentinelFilesCleaner sentinelFilesCleaner,
      BytesDeletedReporter bytesDeletedReporter); @Override @TimedTaggable("s3-paths-deleted") void cleanupPath(HousekeepingEntity housekeepingEntity); }
S3PathCleaner implements PathCleaner { @Override @TimedTaggable("s3-paths-deleted") public void cleanupPath(HousekeepingEntity housekeepingEntity) { S3SchemeURI s3SchemeURI = new S3SchemeURI(housekeepingEntity.getPath()); String key = s3SchemeURI.getKey(); String bucket = s3SchemeURI.getBucket(); S3BytesDeletedCalculator bytesDeletedCalculator = new S3BytesDeletedCalculator(s3Client); boolean isFile = s3Client.doesObjectExist(bucket, key); try { if (isFile) { deleteFile(bucket, key, bytesDeletedCalculator); } else { deleteFilesInDirectory(bucket, key, bytesDeletedCalculator); deleteSentinelFiles(s3SchemeURI, key, bucket, housekeepingEntity.getTableName()); } } finally { long bytesDeleted = bytesDeletedCalculator.getBytesDeleted(); if (bytesDeleted > 0) { bytesDeletedReporter.reportTaggable(bytesDeleted, housekeepingEntity, FileSystemType.S3); } } } S3PathCleaner(S3Client s3Client, SentinelFilesCleaner sentinelFilesCleaner,
      BytesDeletedReporter bytesDeletedReporter); @Override @TimedTaggable("s3-paths-deleted") void cleanupPath(HousekeepingEntity housekeepingEntity);  }