@Test public void testDeleteFolder() throws Exception { String[] originalFiles = new String[]{"a-delete-2.txt", "b-delete-2.txt", "c-delete-2.txt"}; String folderName = "my-files"; for (String fileName : originalFiles) { createEmptyFile(folderName, fileName); } _s3PinotFS.delete(URI.create(String.format(FILE_FORMAT, SCHEME, BUCKET, folderName)), true); ListObjectsV2Response listObjectsV2Response = _s3Client.listObjectsV2(S3TestUtils.getListObjectRequest(BUCKET, "", true)); String[] actualResponse = listObjectsV2Response.contents().stream().map(S3Object::key).filter(x -> x.contains("delete-2")) .toArray(String[]::new); Assert.assertEquals(0, actualResponse.length); }
@Override public boolean delete(URI segmentUri, boolean forceDelete) throws IOException { LOGGER.info("Deleting uri {} force {}", segmentUri, forceDelete); try { if (isDirectory(segmentUri)) { if (!forceDelete) { Preconditions .checkState(isEmptyDirectory(segmentUri), "ForceDelete flag is not set and directory '%s' is not empty", segmentUri); } String prefix = normalizeToDirectoryPrefix(segmentUri); ListObjectsV2Response listObjectsV2Response; ListObjectsV2Request.Builder listObjectsV2RequestBuilder = ListObjectsV2Request.builder().bucket(segmentUri.getHost()); if (prefix.equals(DELIMITER)) { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } else { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.prefix(prefix).build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } boolean deleteSucceeded = true; for (S3Object s3Object : listObjectsV2Response.contents()) { DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(s3Object.key()).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); deleteSucceeded &= deleteObjectResponse.sdkHttpResponse().isSuccessful(); } return deleteSucceeded; } else { String prefix = sanitizePath(segmentUri.getPath()); DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(prefix).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); return deleteObjectResponse.sdkHttpResponse().isSuccessful(); } } catch (NoSuchKeyException e) { return false; } catch (S3Exception e) { throw e; } catch (Exception e) { throw new IOException(e); } }
S3PinotFS extends PinotFS { @Override public boolean delete(URI segmentUri, boolean forceDelete) throws IOException { LOGGER.info("Deleting uri {} force {}", segmentUri, forceDelete); try { if (isDirectory(segmentUri)) { if (!forceDelete) { Preconditions .checkState(isEmptyDirectory(segmentUri), "ForceDelete flag is not set and directory '%s' is not empty", segmentUri); } String prefix = normalizeToDirectoryPrefix(segmentUri); ListObjectsV2Response listObjectsV2Response; ListObjectsV2Request.Builder listObjectsV2RequestBuilder = ListObjectsV2Request.builder().bucket(segmentUri.getHost()); if (prefix.equals(DELIMITER)) { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } else { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.prefix(prefix).build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } boolean deleteSucceeded = true; for (S3Object s3Object : listObjectsV2Response.contents()) { DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(s3Object.key()).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); deleteSucceeded &= deleteObjectResponse.sdkHttpResponse().isSuccessful(); } return deleteSucceeded; } else { String prefix = sanitizePath(segmentUri.getPath()); DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(prefix).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); return deleteObjectResponse.sdkHttpResponse().isSuccessful(); } } catch (NoSuchKeyException e) { return false; } catch (S3Exception e) { throw e; } catch (Exception e) { throw new IOException(e); } } }
S3PinotFS extends PinotFS { @Override public boolean delete(URI segmentUri, boolean forceDelete) throws IOException { LOGGER.info("Deleting uri {} force {}", segmentUri, forceDelete); try { if (isDirectory(segmentUri)) { if (!forceDelete) { Preconditions .checkState(isEmptyDirectory(segmentUri), "ForceDelete flag is not set and directory '%s' is not empty", segmentUri); } String prefix = normalizeToDirectoryPrefix(segmentUri); ListObjectsV2Response listObjectsV2Response; ListObjectsV2Request.Builder listObjectsV2RequestBuilder = ListObjectsV2Request.builder().bucket(segmentUri.getHost()); if (prefix.equals(DELIMITER)) { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } else { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.prefix(prefix).build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } boolean deleteSucceeded = true; for (S3Object s3Object : listObjectsV2Response.contents()) { DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(s3Object.key()).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); deleteSucceeded &= deleteObjectResponse.sdkHttpResponse().isSuccessful(); } return deleteSucceeded; } else { String prefix = sanitizePath(segmentUri.getPath()); DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(prefix).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); return deleteObjectResponse.sdkHttpResponse().isSuccessful(); } } catch (NoSuchKeyException e) { return false; } catch (S3Exception e) { throw e; } catch (Exception e) { throw new IOException(e); } }  }
S3PinotFS extends PinotFS { @Override public boolean delete(URI segmentUri, boolean forceDelete) throws IOException { LOGGER.info("Deleting uri {} force {}", segmentUri, forceDelete); try { if (isDirectory(segmentUri)) { if (!forceDelete) { Preconditions .checkState(isEmptyDirectory(segmentUri), "ForceDelete flag is not set and directory '%s' is not empty", segmentUri); } String prefix = normalizeToDirectoryPrefix(segmentUri); ListObjectsV2Response listObjectsV2Response; ListObjectsV2Request.Builder listObjectsV2RequestBuilder = ListObjectsV2Request.builder().bucket(segmentUri.getHost()); if (prefix.equals(DELIMITER)) { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } else { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.prefix(prefix).build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } boolean deleteSucceeded = true; for (S3Object s3Object : listObjectsV2Response.contents()) { DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(s3Object.key()).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); deleteSucceeded &= deleteObjectResponse.sdkHttpResponse().isSuccessful(); } return deleteSucceeded; } else { String prefix = sanitizePath(segmentUri.getPath()); DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(prefix).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); return deleteObjectResponse.sdkHttpResponse().isSuccessful(); } } catch (NoSuchKeyException e) { return false; } catch (S3Exception e) { throw e; } catch (Exception e) { throw new IOException(e); } }  @Override void init(PinotConfiguration config); void init(S3Client s3Client); @Override boolean mkdir(URI uri); @Override boolean delete(URI segmentUri, boolean forceDelete); @Override boolean doMove(URI srcUri, URI dstUri); @Override boolean copy(URI srcUri, URI dstUri); @Override boolean exists(URI fileUri); @Override long length(URI fileUri); @Override String[] listFiles(URI fileUri, boolean recursive); @Override void copyToLocalFile(URI srcUri, File dstFile); @Override void copyFromLocalFile(File srcFile, URI dstUri); @Override boolean isDirectory(URI uri); @Override long lastModified(URI uri); @Override boolean touch(URI uri); @Override InputStream open(URI uri); @Override void close(); }
S3PinotFS extends PinotFS { @Override public boolean delete(URI segmentUri, boolean forceDelete) throws IOException { LOGGER.info("Deleting uri {} force {}", segmentUri, forceDelete); try { if (isDirectory(segmentUri)) { if (!forceDelete) { Preconditions .checkState(isEmptyDirectory(segmentUri), "ForceDelete flag is not set and directory '%s' is not empty", segmentUri); } String prefix = normalizeToDirectoryPrefix(segmentUri); ListObjectsV2Response listObjectsV2Response; ListObjectsV2Request.Builder listObjectsV2RequestBuilder = ListObjectsV2Request.builder().bucket(segmentUri.getHost()); if (prefix.equals(DELIMITER)) { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } else { ListObjectsV2Request listObjectsV2Request = listObjectsV2RequestBuilder.prefix(prefix).build(); listObjectsV2Response = _s3Client.listObjectsV2(listObjectsV2Request); } boolean deleteSucceeded = true; for (S3Object s3Object : listObjectsV2Response.contents()) { DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(s3Object.key()).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); deleteSucceeded &= deleteObjectResponse.sdkHttpResponse().isSuccessful(); } return deleteSucceeded; } else { String prefix = sanitizePath(segmentUri.getPath()); DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder().bucket(segmentUri.getHost()).key(prefix).build(); DeleteObjectResponse deleteObjectResponse = _s3Client.deleteObject(deleteObjectRequest); return deleteObjectResponse.sdkHttpResponse().isSuccessful(); } } catch (NoSuchKeyException e) { return false; } catch (S3Exception e) { throw e; } catch (Exception e) { throw new IOException(e); } }  @Override void init(PinotConfiguration config); void init(S3Client s3Client); @Override boolean mkdir(URI uri); @Override boolean delete(URI segmentUri, boolean forceDelete); @Override boolean doMove(URI srcUri, URI dstUri); @Override boolean copy(URI srcUri, URI dstUri); @Override boolean exists(URI fileUri); @Override long length(URI fileUri); @Override String[] listFiles(URI fileUri, boolean recursive); @Override void copyToLocalFile(URI srcUri, File dstFile); @Override void copyFromLocalFile(File srcFile, URI dstUri); @Override boolean isDirectory(URI uri); @Override long lastModified(URI uri); @Override boolean touch(URI uri); @Override InputStream open(URI uri); @Override void close(); static final String ACCESS_KEY; static final String SECRET_KEY; static final String REGION; static final String ENDPOINT; static final String S3_SCHEME; }