@Test public void testList() throws Exception { final Path container = new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume)); final AttributedList<Path> list = new S3VersionedObjectListService(session).list( new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume)), new DisabledListProgressListener()); for(Path p : list) { assertEquals(container, p.getParent()); if(p.isFile()) { assertNotEquals(-1L, p.attributes().getModificationDate()); assertNotEquals(-1L, p.attributes().getSize()); assertNotNull(p.attributes().getETag()); assertNotNull(p.attributes().getStorageClass()); assertEquals("null", p.attributes().getVersionId()); } } }
@Override public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("list", concurrency); try { final String prefix = this.createPrefix(directory); final Path bucket = containerService.getContainer(directory); final AttributedList<Path> children = new AttributedList<Path>(); final List<Future<Path>> folders = new ArrayList<Future<Path>>(); String priorLastKey = null; String priorLastVersionId = null; long revision = 0L; String lastKey = null; boolean hasDirectoryPlaceholder = containerService.isContainer(directory); do { final VersionOrDeleteMarkersChunk chunk = session.getClient().listVersionedObjectsChunked( bucket.getName(), prefix, String.valueOf(Path.DELIMITER), preferences.getInteger("s3.listing.chunksize"), priorLastKey, priorLastVersionId, false); for(BaseVersionOrDeleteMarker marker : chunk.getItems()) { final String key = PathNormalizer.normalize(URIEncoder.decode(marker.getKey())); if(String.valueOf(Path.DELIMITER).equals(key)) { log.warn(String.format("Skipping prefix %s", key)); continue; } if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { hasDirectoryPlaceholder = true; continue; } final PathAttributes attributes = new PathAttributes(); attributes.setVersionId(marker.getVersionId()); if(!StringUtils.equals(lastKey, key)) { revision = 0L; } attributes.setRevision(++revision); attributes.setDuplicate(marker.isDeleteMarker() && marker.isLatest() || !marker.isLatest()); if(marker.isDeleteMarker()) { attributes.setCustom(Collections.singletonMap(KEY_DELETE_MARKER, Boolean.TRUE.toString())); } attributes.setModificationDate(marker.getLastModified().getTime()); attributes.setRegion(bucket.attributes().getRegion()); if(marker instanceof S3Version) { final S3Version object = (S3Version) marker; attributes.setSize(object.getSize()); if(StringUtils.isNotBlank(object.getEtag())) { attributes.setETag(object.getEtag()); } if(StringUtils.isNotBlank(object.getStorageClass())) { attributes.setStorageClass(object.getStorageClass()); } } final Path f = new Path(directory.isDirectory() ? directory : directory.getParent(), PathNormalizer.name(key), EnumSet.of(Path.Type.file), attributes); if(references) { if(attributes.isDuplicate()) { final Path latest = children.find(new LatestVersionPathPredicate(f)); if(latest != null) { final AttributedList<Path> versions = new AttributedList<>(latest.attributes().getVersions()); versions.add(f); latest.attributes().setVersions(versions); } else { log.warn(String.format("No current version found for %s", f)); } } } children.add(f); lastKey = key; } final String[] prefixes = chunk.getCommonPrefixes(); for(String common : prefixes) { if(String.valueOf(Path.DELIMITER).equals(common)) { log.warn(String.format("Skipping prefix %s", common)); continue; } final String key = PathNormalizer.normalize(URIEncoder.decode(common)); if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { continue; } folders.add(this.submit(pool, bucket, directory, URIEncoder.decode(common))); } priorLastKey = null != chunk.getNextKeyMarker() ? URIEncoder.decode(chunk.getNextKeyMarker()) : null; priorLastVersionId = chunk.getNextVersionIdMarker(); listener.chunk(directory, children); } while(priorLastKey != null); for(Future<Path> future : folders) { try { children.add(future.get()); } catch(InterruptedException e) { log.error("Listing versioned objects failed with interrupt failure"); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Listing versioned objects failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } listener.chunk(directory, children); if(!hasDirectoryPlaceholder && children.isEmpty()) { throw new NotfoundException(directory.getAbsolute()); } return children; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Listing directory {0} failed", e, directory); } finally { pool.shutdown(false); } }
S3VersionedObjectListService extends S3AbstractListService implements ListService { @Override public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("list", concurrency); try { final String prefix = this.createPrefix(directory); final Path bucket = containerService.getContainer(directory); final AttributedList<Path> children = new AttributedList<Path>(); final List<Future<Path>> folders = new ArrayList<Future<Path>>(); String priorLastKey = null; String priorLastVersionId = null; long revision = 0L; String lastKey = null; boolean hasDirectoryPlaceholder = containerService.isContainer(directory); do { final VersionOrDeleteMarkersChunk chunk = session.getClient().listVersionedObjectsChunked( bucket.getName(), prefix, String.valueOf(Path.DELIMITER), preferences.getInteger("s3.listing.chunksize"), priorLastKey, priorLastVersionId, false); for(BaseVersionOrDeleteMarker marker : chunk.getItems()) { final String key = PathNormalizer.normalize(URIEncoder.decode(marker.getKey())); if(String.valueOf(Path.DELIMITER).equals(key)) { log.warn(String.format("Skipping prefix %s", key)); continue; } if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { hasDirectoryPlaceholder = true; continue; } final PathAttributes attributes = new PathAttributes(); attributes.setVersionId(marker.getVersionId()); if(!StringUtils.equals(lastKey, key)) { revision = 0L; } attributes.setRevision(++revision); attributes.setDuplicate(marker.isDeleteMarker() && marker.isLatest() || !marker.isLatest()); if(marker.isDeleteMarker()) { attributes.setCustom(Collections.singletonMap(KEY_DELETE_MARKER, Boolean.TRUE.toString())); } attributes.setModificationDate(marker.getLastModified().getTime()); attributes.setRegion(bucket.attributes().getRegion()); if(marker instanceof S3Version) { final S3Version object = (S3Version) marker; attributes.setSize(object.getSize()); if(StringUtils.isNotBlank(object.getEtag())) { attributes.setETag(object.getEtag()); } if(StringUtils.isNotBlank(object.getStorageClass())) { attributes.setStorageClass(object.getStorageClass()); } } final Path f = new Path(directory.isDirectory() ? directory : directory.getParent(), PathNormalizer.name(key), EnumSet.of(Path.Type.file), attributes); if(references) { if(attributes.isDuplicate()) { final Path latest = children.find(new LatestVersionPathPredicate(f)); if(latest != null) { final AttributedList<Path> versions = new AttributedList<>(latest.attributes().getVersions()); versions.add(f); latest.attributes().setVersions(versions); } else { log.warn(String.format("No current version found for %s", f)); } } } children.add(f); lastKey = key; } final String[] prefixes = chunk.getCommonPrefixes(); for(String common : prefixes) { if(String.valueOf(Path.DELIMITER).equals(common)) { log.warn(String.format("Skipping prefix %s", common)); continue; } final String key = PathNormalizer.normalize(URIEncoder.decode(common)); if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { continue; } folders.add(this.submit(pool, bucket, directory, URIEncoder.decode(common))); } priorLastKey = null != chunk.getNextKeyMarker() ? URIEncoder.decode(chunk.getNextKeyMarker()) : null; priorLastVersionId = chunk.getNextVersionIdMarker(); listener.chunk(directory, children); } while(priorLastKey != null); for(Future<Path> future : folders) { try { children.add(future.get()); } catch(InterruptedException e) { log.error("Listing versioned objects failed with interrupt failure"); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Listing versioned objects failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } listener.chunk(directory, children); if(!hasDirectoryPlaceholder && children.isEmpty()) { throw new NotfoundException(directory.getAbsolute()); } return children; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Listing directory {0} failed", e, directory); } finally { pool.shutdown(false); } } }
S3VersionedObjectListService extends S3AbstractListService implements ListService { @Override public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("list", concurrency); try { final String prefix = this.createPrefix(directory); final Path bucket = containerService.getContainer(directory); final AttributedList<Path> children = new AttributedList<Path>(); final List<Future<Path>> folders = new ArrayList<Future<Path>>(); String priorLastKey = null; String priorLastVersionId = null; long revision = 0L; String lastKey = null; boolean hasDirectoryPlaceholder = containerService.isContainer(directory); do { final VersionOrDeleteMarkersChunk chunk = session.getClient().listVersionedObjectsChunked( bucket.getName(), prefix, String.valueOf(Path.DELIMITER), preferences.getInteger("s3.listing.chunksize"), priorLastKey, priorLastVersionId, false); for(BaseVersionOrDeleteMarker marker : chunk.getItems()) { final String key = PathNormalizer.normalize(URIEncoder.decode(marker.getKey())); if(String.valueOf(Path.DELIMITER).equals(key)) { log.warn(String.format("Skipping prefix %s", key)); continue; } if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { hasDirectoryPlaceholder = true; continue; } final PathAttributes attributes = new PathAttributes(); attributes.setVersionId(marker.getVersionId()); if(!StringUtils.equals(lastKey, key)) { revision = 0L; } attributes.setRevision(++revision); attributes.setDuplicate(marker.isDeleteMarker() && marker.isLatest() || !marker.isLatest()); if(marker.isDeleteMarker()) { attributes.setCustom(Collections.singletonMap(KEY_DELETE_MARKER, Boolean.TRUE.toString())); } attributes.setModificationDate(marker.getLastModified().getTime()); attributes.setRegion(bucket.attributes().getRegion()); if(marker instanceof S3Version) { final S3Version object = (S3Version) marker; attributes.setSize(object.getSize()); if(StringUtils.isNotBlank(object.getEtag())) { attributes.setETag(object.getEtag()); } if(StringUtils.isNotBlank(object.getStorageClass())) { attributes.setStorageClass(object.getStorageClass()); } } final Path f = new Path(directory.isDirectory() ? directory : directory.getParent(), PathNormalizer.name(key), EnumSet.of(Path.Type.file), attributes); if(references) { if(attributes.isDuplicate()) { final Path latest = children.find(new LatestVersionPathPredicate(f)); if(latest != null) { final AttributedList<Path> versions = new AttributedList<>(latest.attributes().getVersions()); versions.add(f); latest.attributes().setVersions(versions); } else { log.warn(String.format("No current version found for %s", f)); } } } children.add(f); lastKey = key; } final String[] prefixes = chunk.getCommonPrefixes(); for(String common : prefixes) { if(String.valueOf(Path.DELIMITER).equals(common)) { log.warn(String.format("Skipping prefix %s", common)); continue; } final String key = PathNormalizer.normalize(URIEncoder.decode(common)); if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { continue; } folders.add(this.submit(pool, bucket, directory, URIEncoder.decode(common))); } priorLastKey = null != chunk.getNextKeyMarker() ? URIEncoder.decode(chunk.getNextKeyMarker()) : null; priorLastVersionId = chunk.getNextVersionIdMarker(); listener.chunk(directory, children); } while(priorLastKey != null); for(Future<Path> future : folders) { try { children.add(future.get()); } catch(InterruptedException e) { log.error("Listing versioned objects failed with interrupt failure"); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Listing versioned objects failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } listener.chunk(directory, children); if(!hasDirectoryPlaceholder && children.isEmpty()) { throw new NotfoundException(directory.getAbsolute()); } return children; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Listing directory {0} failed", e, directory); } finally { pool.shutdown(false); } } S3VersionedObjectListService(final S3Session session);  S3VersionedObjectListService(final S3Session session, final boolean references);  S3VersionedObjectListService(final S3Session session, final Integer concurrency, final boolean references); }
S3VersionedObjectListService extends S3AbstractListService implements ListService { @Override public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("list", concurrency); try { final String prefix = this.createPrefix(directory); final Path bucket = containerService.getContainer(directory); final AttributedList<Path> children = new AttributedList<Path>(); final List<Future<Path>> folders = new ArrayList<Future<Path>>(); String priorLastKey = null; String priorLastVersionId = null; long revision = 0L; String lastKey = null; boolean hasDirectoryPlaceholder = containerService.isContainer(directory); do { final VersionOrDeleteMarkersChunk chunk = session.getClient().listVersionedObjectsChunked( bucket.getName(), prefix, String.valueOf(Path.DELIMITER), preferences.getInteger("s3.listing.chunksize"), priorLastKey, priorLastVersionId, false); for(BaseVersionOrDeleteMarker marker : chunk.getItems()) { final String key = PathNormalizer.normalize(URIEncoder.decode(marker.getKey())); if(String.valueOf(Path.DELIMITER).equals(key)) { log.warn(String.format("Skipping prefix %s", key)); continue; } if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { hasDirectoryPlaceholder = true; continue; } final PathAttributes attributes = new PathAttributes(); attributes.setVersionId(marker.getVersionId()); if(!StringUtils.equals(lastKey, key)) { revision = 0L; } attributes.setRevision(++revision); attributes.setDuplicate(marker.isDeleteMarker() && marker.isLatest() || !marker.isLatest()); if(marker.isDeleteMarker()) { attributes.setCustom(Collections.singletonMap(KEY_DELETE_MARKER, Boolean.TRUE.toString())); } attributes.setModificationDate(marker.getLastModified().getTime()); attributes.setRegion(bucket.attributes().getRegion()); if(marker instanceof S3Version) { final S3Version object = (S3Version) marker; attributes.setSize(object.getSize()); if(StringUtils.isNotBlank(object.getEtag())) { attributes.setETag(object.getEtag()); } if(StringUtils.isNotBlank(object.getStorageClass())) { attributes.setStorageClass(object.getStorageClass()); } } final Path f = new Path(directory.isDirectory() ? directory : directory.getParent(), PathNormalizer.name(key), EnumSet.of(Path.Type.file), attributes); if(references) { if(attributes.isDuplicate()) { final Path latest = children.find(new LatestVersionPathPredicate(f)); if(latest != null) { final AttributedList<Path> versions = new AttributedList<>(latest.attributes().getVersions()); versions.add(f); latest.attributes().setVersions(versions); } else { log.warn(String.format("No current version found for %s", f)); } } } children.add(f); lastKey = key; } final String[] prefixes = chunk.getCommonPrefixes(); for(String common : prefixes) { if(String.valueOf(Path.DELIMITER).equals(common)) { log.warn(String.format("Skipping prefix %s", common)); continue; } final String key = PathNormalizer.normalize(URIEncoder.decode(common)); if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { continue; } folders.add(this.submit(pool, bucket, directory, URIEncoder.decode(common))); } priorLastKey = null != chunk.getNextKeyMarker() ? URIEncoder.decode(chunk.getNextKeyMarker()) : null; priorLastVersionId = chunk.getNextVersionIdMarker(); listener.chunk(directory, children); } while(priorLastKey != null); for(Future<Path> future : folders) { try { children.add(future.get()); } catch(InterruptedException e) { log.error("Listing versioned objects failed with interrupt failure"); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Listing versioned objects failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } listener.chunk(directory, children); if(!hasDirectoryPlaceholder && children.isEmpty()) { throw new NotfoundException(directory.getAbsolute()); } return children; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Listing directory {0} failed", e, directory); } finally { pool.shutdown(false); } } S3VersionedObjectListService(final S3Session session);  S3VersionedObjectListService(final S3Session session, final boolean references);  S3VersionedObjectListService(final S3Session session, final Integer concurrency, final boolean references); @Override AttributedList<Path> list(final Path directory, final ListProgressListener listener); }
S3VersionedObjectListService extends S3AbstractListService implements ListService { @Override public AttributedList<Path> list(final Path directory, final ListProgressListener listener) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("list", concurrency); try { final String prefix = this.createPrefix(directory); final Path bucket = containerService.getContainer(directory); final AttributedList<Path> children = new AttributedList<Path>(); final List<Future<Path>> folders = new ArrayList<Future<Path>>(); String priorLastKey = null; String priorLastVersionId = null; long revision = 0L; String lastKey = null; boolean hasDirectoryPlaceholder = containerService.isContainer(directory); do { final VersionOrDeleteMarkersChunk chunk = session.getClient().listVersionedObjectsChunked( bucket.getName(), prefix, String.valueOf(Path.DELIMITER), preferences.getInteger("s3.listing.chunksize"), priorLastKey, priorLastVersionId, false); for(BaseVersionOrDeleteMarker marker : chunk.getItems()) { final String key = PathNormalizer.normalize(URIEncoder.decode(marker.getKey())); if(String.valueOf(Path.DELIMITER).equals(key)) { log.warn(String.format("Skipping prefix %s", key)); continue; } if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { hasDirectoryPlaceholder = true; continue; } final PathAttributes attributes = new PathAttributes(); attributes.setVersionId(marker.getVersionId()); if(!StringUtils.equals(lastKey, key)) { revision = 0L; } attributes.setRevision(++revision); attributes.setDuplicate(marker.isDeleteMarker() && marker.isLatest() || !marker.isLatest()); if(marker.isDeleteMarker()) { attributes.setCustom(Collections.singletonMap(KEY_DELETE_MARKER, Boolean.TRUE.toString())); } attributes.setModificationDate(marker.getLastModified().getTime()); attributes.setRegion(bucket.attributes().getRegion()); if(marker instanceof S3Version) { final S3Version object = (S3Version) marker; attributes.setSize(object.getSize()); if(StringUtils.isNotBlank(object.getEtag())) { attributes.setETag(object.getEtag()); } if(StringUtils.isNotBlank(object.getStorageClass())) { attributes.setStorageClass(object.getStorageClass()); } } final Path f = new Path(directory.isDirectory() ? directory : directory.getParent(), PathNormalizer.name(key), EnumSet.of(Path.Type.file), attributes); if(references) { if(attributes.isDuplicate()) { final Path latest = children.find(new LatestVersionPathPredicate(f)); if(latest != null) { final AttributedList<Path> versions = new AttributedList<>(latest.attributes().getVersions()); versions.add(f); latest.attributes().setVersions(versions); } else { log.warn(String.format("No current version found for %s", f)); } } } children.add(f); lastKey = key; } final String[] prefixes = chunk.getCommonPrefixes(); for(String common : prefixes) { if(String.valueOf(Path.DELIMITER).equals(common)) { log.warn(String.format("Skipping prefix %s", common)); continue; } final String key = PathNormalizer.normalize(URIEncoder.decode(common)); if(new Path(bucket, key, EnumSet.of(Path.Type.directory)).equals(directory)) { continue; } folders.add(this.submit(pool, bucket, directory, URIEncoder.decode(common))); } priorLastKey = null != chunk.getNextKeyMarker() ? URIEncoder.decode(chunk.getNextKeyMarker()) : null; priorLastVersionId = chunk.getNextVersionIdMarker(); listener.chunk(directory, children); } while(priorLastKey != null); for(Future<Path> future : folders) { try { children.add(future.get()); } catch(InterruptedException e) { log.error("Listing versioned objects failed with interrupt failure"); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Listing versioned objects failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } listener.chunk(directory, children); if(!hasDirectoryPlaceholder && children.isEmpty()) { throw new NotfoundException(directory.getAbsolute()); } return children; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Listing directory {0} failed", e, directory); } finally { pool.shutdown(false); } } S3VersionedObjectListService(final S3Session session);  S3VersionedObjectListService(final S3Session session, final boolean references);  S3VersionedObjectListService(final S3Session session, final Integer concurrency, final boolean references); @Override AttributedList<Path> list(final Path directory, final ListProgressListener listener); static final String KEY_DELETE_MARKER; }