@Test public void testAppendNoPartCompleted() throws Exception { final Path container = new Path("test-us-east-1-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume)); String name = UUID.randomUUID().toString(); final Path test = new Path(container, name, EnumSet.of(Path.Type.file)); final Local local = new Local(System.getProperty("java.io.tmpdir"), name); final int length = 32769; final byte[] content = RandomUtils.nextBytes(length); IOUtils.write(content, local.getOutputStream(false)); final AtomicBoolean started = new AtomicBoolean(); final TransferStatus status = new TransferStatus() { @Override public void progress(long bytes) { super.progress(bytes); started.set(true); } }; status.setLength(content.length); final AtomicBoolean interrupt = new AtomicBoolean(); try { new S3MultipartUploadService(session, new S3WriteFeature(session, new S3DisabledMultipartService()), 10485760L, 1).upload(test, new Local(System.getProperty("java.io.tmpdir"), name) { @Override public InputStream getInputStream() throws AccessDeniedException { return new CountingInputStream(super.getInputStream()) { @Override protected void beforeRead(int n) throws IOException { if(started.get()) { if(this.getByteCount() >= 32768) { throw new IOException(); } } } }; } }, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), status, new DisabledConnectionCallback()); } catch(BackgroundException e) { interrupt.set(true); } assertTrue(interrupt.get()); assertEquals(0L, status.getOffset(), 0L); assertFalse(status.isComplete()); final TransferStatus append = new TransferStatus().append(true).length(content.length); new S3MultipartUploadService(session, new S3WriteFeature(session, new S3DisabledMultipartService()), 10485760L, 1).upload( test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), append, new DisabledConnectionCallback()); assertEquals(32769L, append.getOffset(), 0L); assertTrue(append.isComplete()); assertTrue(new S3FindFeature(session).find(test)); assertEquals(content.length, new S3AttributesFinderFeature(session).find(test).getSize()); final byte[] buffer = new byte[content.length]; final InputStream in = new S3ReadFeature(session).read(test, new TransferStatus(), new DisabledConnectionCallback()); IOUtils.readFully(in, buffer); in.close(); assertArrayEquals(content, buffer); new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); local.delete(); }
@Override public StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("multipart", concurrency); try { MultipartUpload multipart = null; try { if(status.isAppend()) { final List<MultipartUpload> list = multipartService.find(file); if(!list.isEmpty()) { multipart = list.iterator().next(); } } } catch(AccessDeniedException | InteroperabilityException e) { log.warn(String.format("Ignore failure listing incomplete multipart uploads. %s", e)); } final List<MultipartPart> completed = new ArrayList<>(); if(null == multipart) { if(log.isInfoEnabled()) { log.info("No pending multipart upload found"); } final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService()) .getDetails(file, status); multipart = session.getClient().multipartStartUpload(containerService.getContainer(file).getName(), object); if(log.isDebugEnabled()) { log.debug(String.format("Multipart upload started for %s with ID %s", multipart.getObjectKey(), multipart.getUploadId())); } } else { if(status.isAppend()) { completed.addAll(multipartService.list(multipart)); } } final long size = status.getLength() + status.getOffset(); final List<Future<MultipartPart>> parts = new ArrayList<>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(MultipartPart c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getSize(); break; } } } if(!skip) { final long length = Math.min(Math.max((size / (S3DefaultMultipartService.MAXIMUM_UPLOAD_PARTS - 1)), partsize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, multipart, partNumber, offset, length, callback)); remaining -= length; offset += length; } } for(Future<MultipartPart> future : parts) { try { completed.add(future.get()); } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } final MultipartCompleted complete = session.getClient().multipartCompleteUpload(multipart, completed); if(log.isInfoEnabled()) { log.info(String.format("Completed multipart upload for %s with %d parts and checksum %s", complete.getObjectKey(), completed.size(), complete.getEtag())); } if(file.getType().contains(Path.Type.encrypted)) { log.warn(String.format("Skip checksum verification for %s with client side encryption enabled", file)); } else { if(S3Session.isAwsHostname(session.getHost().getHostname())) { completed.sort(new MultipartPart.PartNumberComparator()); final StringBuilder concat = new StringBuilder(); for(MultipartPart part : completed) { concat.append(part.getEtag()); } final String expected = String.format("%s-%d", ChecksumComputeFactory.get(HashAlgorithm.md5).compute(concat.toString(), new TransferStatus()), completed.size()); final String reference; if(complete.getEtag().startsWith("\"") && complete.getEtag().endsWith("\"")) { reference = complete.getEtag().substring(1, complete.getEtag().length() - 1); } else { reference = complete.getEtag(); } if(!StringUtils.equalsIgnoreCase(expected, reference)) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between MD5 hash {0} of uploaded data and ETag {1} returned by the server", expected, reference)); } } } status.setComplete(); final StorageObject object = new StorageObject(containerService.getKey(file)); object.setETag(complete.getEtag()); return object; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } }
S3MultipartUploadService extends HttpUploadFeature<StorageObject, MessageDigest> { @Override public StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("multipart", concurrency); try { MultipartUpload multipart = null; try { if(status.isAppend()) { final List<MultipartUpload> list = multipartService.find(file); if(!list.isEmpty()) { multipart = list.iterator().next(); } } } catch(AccessDeniedException | InteroperabilityException e) { log.warn(String.format("Ignore failure listing incomplete multipart uploads. %s", e)); } final List<MultipartPart> completed = new ArrayList<>(); if(null == multipart) { if(log.isInfoEnabled()) { log.info("No pending multipart upload found"); } final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService()) .getDetails(file, status); multipart = session.getClient().multipartStartUpload(containerService.getContainer(file).getName(), object); if(log.isDebugEnabled()) { log.debug(String.format("Multipart upload started for %s with ID %s", multipart.getObjectKey(), multipart.getUploadId())); } } else { if(status.isAppend()) { completed.addAll(multipartService.list(multipart)); } } final long size = status.getLength() + status.getOffset(); final List<Future<MultipartPart>> parts = new ArrayList<>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(MultipartPart c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getSize(); break; } } } if(!skip) { final long length = Math.min(Math.max((size / (S3DefaultMultipartService.MAXIMUM_UPLOAD_PARTS - 1)), partsize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, multipart, partNumber, offset, length, callback)); remaining -= length; offset += length; } } for(Future<MultipartPart> future : parts) { try { completed.add(future.get()); } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } final MultipartCompleted complete = session.getClient().multipartCompleteUpload(multipart, completed); if(log.isInfoEnabled()) { log.info(String.format("Completed multipart upload for %s with %d parts and checksum %s", complete.getObjectKey(), completed.size(), complete.getEtag())); } if(file.getType().contains(Path.Type.encrypted)) { log.warn(String.format("Skip checksum verification for %s with client side encryption enabled", file)); } else { if(S3Session.isAwsHostname(session.getHost().getHostname())) { completed.sort(new MultipartPart.PartNumberComparator()); final StringBuilder concat = new StringBuilder(); for(MultipartPart part : completed) { concat.append(part.getEtag()); } final String expected = String.format("%s-%d", ChecksumComputeFactory.get(HashAlgorithm.md5).compute(concat.toString(), new TransferStatus()), completed.size()); final String reference; if(complete.getEtag().startsWith("\"") && complete.getEtag().endsWith("\"")) { reference = complete.getEtag().substring(1, complete.getEtag().length() - 1); } else { reference = complete.getEtag(); } if(!StringUtils.equalsIgnoreCase(expected, reference)) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between MD5 hash {0} of uploaded data and ETag {1} returned by the server", expected, reference)); } } } status.setComplete(); final StorageObject object = new StorageObject(containerService.getKey(file)); object.setETag(complete.getEtag()); return object; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } }
S3MultipartUploadService extends HttpUploadFeature<StorageObject, MessageDigest> { @Override public StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("multipart", concurrency); try { MultipartUpload multipart = null; try { if(status.isAppend()) { final List<MultipartUpload> list = multipartService.find(file); if(!list.isEmpty()) { multipart = list.iterator().next(); } } } catch(AccessDeniedException | InteroperabilityException e) { log.warn(String.format("Ignore failure listing incomplete multipart uploads. %s", e)); } final List<MultipartPart> completed = new ArrayList<>(); if(null == multipart) { if(log.isInfoEnabled()) { log.info("No pending multipart upload found"); } final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService()) .getDetails(file, status); multipart = session.getClient().multipartStartUpload(containerService.getContainer(file).getName(), object); if(log.isDebugEnabled()) { log.debug(String.format("Multipart upload started for %s with ID %s", multipart.getObjectKey(), multipart.getUploadId())); } } else { if(status.isAppend()) { completed.addAll(multipartService.list(multipart)); } } final long size = status.getLength() + status.getOffset(); final List<Future<MultipartPart>> parts = new ArrayList<>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(MultipartPart c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getSize(); break; } } } if(!skip) { final long length = Math.min(Math.max((size / (S3DefaultMultipartService.MAXIMUM_UPLOAD_PARTS - 1)), partsize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, multipart, partNumber, offset, length, callback)); remaining -= length; offset += length; } } for(Future<MultipartPart> future : parts) { try { completed.add(future.get()); } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } final MultipartCompleted complete = session.getClient().multipartCompleteUpload(multipart, completed); if(log.isInfoEnabled()) { log.info(String.format("Completed multipart upload for %s with %d parts and checksum %s", complete.getObjectKey(), completed.size(), complete.getEtag())); } if(file.getType().contains(Path.Type.encrypted)) { log.warn(String.format("Skip checksum verification for %s with client side encryption enabled", file)); } else { if(S3Session.isAwsHostname(session.getHost().getHostname())) { completed.sort(new MultipartPart.PartNumberComparator()); final StringBuilder concat = new StringBuilder(); for(MultipartPart part : completed) { concat.append(part.getEtag()); } final String expected = String.format("%s-%d", ChecksumComputeFactory.get(HashAlgorithm.md5).compute(concat.toString(), new TransferStatus()), completed.size()); final String reference; if(complete.getEtag().startsWith("\"") && complete.getEtag().endsWith("\"")) { reference = complete.getEtag().substring(1, complete.getEtag().length() - 1); } else { reference = complete.getEtag(); } if(!StringUtils.equalsIgnoreCase(expected, reference)) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between MD5 hash {0} of uploaded data and ETag {1} returned by the server", expected, reference)); } } } status.setComplete(); final StorageObject object = new StorageObject(containerService.getKey(file)); object.setETag(complete.getEtag()); return object; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } S3MultipartUploadService(final S3Session session, final Write<StorageObject> writer);  S3MultipartUploadService(final S3Session session, final Write<StorageObject> writer, final Long partsize, final Integer concurrency); }
S3MultipartUploadService extends HttpUploadFeature<StorageObject, MessageDigest> { @Override public StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("multipart", concurrency); try { MultipartUpload multipart = null; try { if(status.isAppend()) { final List<MultipartUpload> list = multipartService.find(file); if(!list.isEmpty()) { multipart = list.iterator().next(); } } } catch(AccessDeniedException | InteroperabilityException e) { log.warn(String.format("Ignore failure listing incomplete multipart uploads. %s", e)); } final List<MultipartPart> completed = new ArrayList<>(); if(null == multipart) { if(log.isInfoEnabled()) { log.info("No pending multipart upload found"); } final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService()) .getDetails(file, status); multipart = session.getClient().multipartStartUpload(containerService.getContainer(file).getName(), object); if(log.isDebugEnabled()) { log.debug(String.format("Multipart upload started for %s with ID %s", multipart.getObjectKey(), multipart.getUploadId())); } } else { if(status.isAppend()) { completed.addAll(multipartService.list(multipart)); } } final long size = status.getLength() + status.getOffset(); final List<Future<MultipartPart>> parts = new ArrayList<>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(MultipartPart c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getSize(); break; } } } if(!skip) { final long length = Math.min(Math.max((size / (S3DefaultMultipartService.MAXIMUM_UPLOAD_PARTS - 1)), partsize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, multipart, partNumber, offset, length, callback)); remaining -= length; offset += length; } } for(Future<MultipartPart> future : parts) { try { completed.add(future.get()); } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } final MultipartCompleted complete = session.getClient().multipartCompleteUpload(multipart, completed); if(log.isInfoEnabled()) { log.info(String.format("Completed multipart upload for %s with %d parts and checksum %s", complete.getObjectKey(), completed.size(), complete.getEtag())); } if(file.getType().contains(Path.Type.encrypted)) { log.warn(String.format("Skip checksum verification for %s with client side encryption enabled", file)); } else { if(S3Session.isAwsHostname(session.getHost().getHostname())) { completed.sort(new MultipartPart.PartNumberComparator()); final StringBuilder concat = new StringBuilder(); for(MultipartPart part : completed) { concat.append(part.getEtag()); } final String expected = String.format("%s-%d", ChecksumComputeFactory.get(HashAlgorithm.md5).compute(concat.toString(), new TransferStatus()), completed.size()); final String reference; if(complete.getEtag().startsWith("\"") && complete.getEtag().endsWith("\"")) { reference = complete.getEtag().substring(1, complete.getEtag().length() - 1); } else { reference = complete.getEtag(); } if(!StringUtils.equalsIgnoreCase(expected, reference)) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between MD5 hash {0} of uploaded data and ETag {1} returned by the server", expected, reference)); } } } status.setComplete(); final StorageObject object = new StorageObject(containerService.getKey(file)); object.setETag(complete.getEtag()); return object; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } S3MultipartUploadService(final S3Session session, final Write<StorageObject> writer);  S3MultipartUploadService(final S3Session session, final Write<StorageObject> writer, final Long partsize, final Integer concurrency); @Override StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener,
                                final TransferStatus status, final ConnectionCallback callback); @Override Upload<StorageObject> withWriter(final Write<StorageObject> writer); }
S3MultipartUploadService extends HttpUploadFeature<StorageObject, MessageDigest> { @Override public StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("multipart", concurrency); try { MultipartUpload multipart = null; try { if(status.isAppend()) { final List<MultipartUpload> list = multipartService.find(file); if(!list.isEmpty()) { multipart = list.iterator().next(); } } } catch(AccessDeniedException | InteroperabilityException e) { log.warn(String.format("Ignore failure listing incomplete multipart uploads. %s", e)); } final List<MultipartPart> completed = new ArrayList<>(); if(null == multipart) { if(log.isInfoEnabled()) { log.info("No pending multipart upload found"); } final S3Object object = new S3WriteFeature(session, new S3DisabledMultipartService()) .getDetails(file, status); multipart = session.getClient().multipartStartUpload(containerService.getContainer(file).getName(), object); if(log.isDebugEnabled()) { log.debug(String.format("Multipart upload started for %s with ID %s", multipart.getObjectKey(), multipart.getUploadId())); } } else { if(status.isAppend()) { completed.addAll(multipartService.list(multipart)); } } final long size = status.getLength() + status.getOffset(); final List<Future<MultipartPart>> parts = new ArrayList<>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(MultipartPart c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getSize(); break; } } } if(!skip) { final long length = Math.min(Math.max((size / (S3DefaultMultipartService.MAXIMUM_UPLOAD_PARTS - 1)), partsize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, multipart, partNumber, offset, length, callback)); remaining -= length; offset += length; } } for(Future<MultipartPart> future : parts) { try { completed.add(future.get()); } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new BackgroundException(e.getCause()); } } final MultipartCompleted complete = session.getClient().multipartCompleteUpload(multipart, completed); if(log.isInfoEnabled()) { log.info(String.format("Completed multipart upload for %s with %d parts and checksum %s", complete.getObjectKey(), completed.size(), complete.getEtag())); } if(file.getType().contains(Path.Type.encrypted)) { log.warn(String.format("Skip checksum verification for %s with client side encryption enabled", file)); } else { if(S3Session.isAwsHostname(session.getHost().getHostname())) { completed.sort(new MultipartPart.PartNumberComparator()); final StringBuilder concat = new StringBuilder(); for(MultipartPart part : completed) { concat.append(part.getEtag()); } final String expected = String.format("%s-%d", ChecksumComputeFactory.get(HashAlgorithm.md5).compute(concat.toString(), new TransferStatus()), completed.size()); final String reference; if(complete.getEtag().startsWith("\"") && complete.getEtag().endsWith("\"")) { reference = complete.getEtag().substring(1, complete.getEtag().length() - 1); } else { reference = complete.getEtag(); } if(!StringUtils.equalsIgnoreCase(expected, reference)) { throw new ChecksumException(MessageFormat.format(LocaleFactory.localizedString("Upload {0} failed", "Error"), file.getName()), MessageFormat.format("Mismatch between MD5 hash {0} of uploaded data and ETag {1} returned by the server", expected, reference)); } } } status.setComplete(); final StorageObject object = new StorageObject(containerService.getKey(file)); object.setETag(complete.getEtag()); return object; } catch(ServiceException e) { throw new S3ExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } S3MultipartUploadService(final S3Session session, final Write<StorageObject> writer);  S3MultipartUploadService(final S3Session session, final Write<StorageObject> writer, final Long partsize, final Integer concurrency); @Override StorageObject upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener,
                                final TransferStatus status, final ConnectionCallback callback); @Override Upload<StorageObject> withWriter(final Write<StorageObject> writer);  }