@Test public void testAppendSecondPart() throws Exception { final Path bucket = new Path("test-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume)); final Path test = new Path(bucket, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file)); final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString()); final int length = 102 * 1024 * 1024; final byte[] content = RandomUtils.nextBytes(length); IOUtils.write(content, local.getOutputStream(false)); final TransferStatus status = new TransferStatus(); status.setLength(content.length); final AtomicBoolean interrupt = new AtomicBoolean(); final B2FileidProvider fileid = new B2FileidProvider(session).withCache(cache); try { new B2LargeUploadService(session, fileid, new B2WriteFeature(session, fileid), 100L * 1024L * 1024L, 1).upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener() { long count; @Override public void sent(final long bytes) { count += bytes; if(count >= 101L * 1024L * 1024L) { throw new RuntimeException(); } } }, status, new DisabledLoginCallback()); } catch(BackgroundException e) { interrupt.set(true); } assertTrue(interrupt.get()); assertEquals(100L * 1024L * 1024L, status.getOffset(), 0L); assertFalse(status.isComplete()); final TransferStatus append = new TransferStatus().append(true).length(2L * 1024L * 1024L).skip(100L * 1024L * 1024L); new B2LargeUploadService(session, fileid, new B2WriteFeature(session, fileid), 100L * 1024L * 1024L, 1).upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), append, new DisabledLoginCallback()); assertEquals(102L * 1024L * 1024L, append.getOffset(), 0L); assertTrue(append.isComplete()); assertTrue(new B2FindFeature(session, fileid).find(test)); assertEquals(102L * 1024L * 1024L, new B2AttributesFinderFeature(session, fileid).find(test).getSize(), 0L); final byte[] buffer = new byte[content.length]; final InputStream in = new B2ReadFeature(session, fileid).read(test, new TransferStatus(), new DisabledConnectionCallback()); IOUtils.readFully(in, buffer); in.close(); assertArrayEquals(content, buffer); new B2DeleteFeature(session, fileid).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback()); local.delete(); }
@Override public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } if(status.isAppend()) { final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid); final List<B2FileInfoResponse> uploads = partService.find(file); if(uploads.isEmpty()) { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } else { status.setVersion(new VersionId(uploads.iterator().next().getFileId())); completed.addAll(partService.list(status.getVersion().id)); } } else { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } final long size = status.getLength() + status.getOffset(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(B2UploadPartResponse c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getContentLength(); break; } } } if(!skip) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } final B2FinishLargeFileResponse response = session.getClient().finishLargeFileUpload(status.getVersion().id, checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", file, completed.size())); } status.setComplete(); return response; } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Upload {0} failed", e, file); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } }
B2LargeUploadService extends HttpUploadFeature<BaseB2Response, MessageDigest> { @Override public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } if(status.isAppend()) { final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid); final List<B2FileInfoResponse> uploads = partService.find(file); if(uploads.isEmpty()) { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } else { status.setVersion(new VersionId(uploads.iterator().next().getFileId())); completed.addAll(partService.list(status.getVersion().id)); } } else { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } final long size = status.getLength() + status.getOffset(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(B2UploadPartResponse c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getContentLength(); break; } } } if(!skip) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } final B2FinishLargeFileResponse response = session.getClient().finishLargeFileUpload(status.getVersion().id, checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", file, completed.size())); } status.setComplete(); return response; } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Upload {0} failed", e, file); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } }
B2LargeUploadService extends HttpUploadFeature<BaseB2Response, MessageDigest> { @Override public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } if(status.isAppend()) { final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid); final List<B2FileInfoResponse> uploads = partService.find(file); if(uploads.isEmpty()) { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } else { status.setVersion(new VersionId(uploads.iterator().next().getFileId())); completed.addAll(partService.list(status.getVersion().id)); } } else { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } final long size = status.getLength() + status.getOffset(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(B2UploadPartResponse c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getContentLength(); break; } } } if(!skip) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } final B2FinishLargeFileResponse response = session.getClient().finishLargeFileUpload(status.getVersion().id, checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", file, completed.size())); } status.setComplete(); return response; } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Upload {0} failed", e, file); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } B2LargeUploadService(final B2Session session, final B2FileidProvider fileid, final Write<BaseB2Response> writer);  B2LargeUploadService(final B2Session session, final B2FileidProvider fileid, final Write<BaseB2Response> writer, final Long partSize, final Integer concurrency); }
B2LargeUploadService extends HttpUploadFeature<BaseB2Response, MessageDigest> { @Override public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } if(status.isAppend()) { final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid); final List<B2FileInfoResponse> uploads = partService.find(file); if(uploads.isEmpty()) { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } else { status.setVersion(new VersionId(uploads.iterator().next().getFileId())); completed.addAll(partService.list(status.getVersion().id)); } } else { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } final long size = status.getLength() + status.getOffset(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(B2UploadPartResponse c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getContentLength(); break; } } } if(!skip) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } final B2FinishLargeFileResponse response = session.getClient().finishLargeFileUpload(status.getVersion().id, checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", file, completed.size())); } status.setComplete(); return response; } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Upload {0} failed", e, file); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } B2LargeUploadService(final B2Session session, final B2FileidProvider fileid, final Write<BaseB2Response> writer);  B2LargeUploadService(final B2Session session, final B2FileidProvider fileid, final Write<BaseB2Response> writer, final Long partSize, final Integer concurrency); @Override BaseB2Response upload(final Path file, final Local local,
                                 final BandwidthThrottle throttle,
                                 final StreamListener listener,
                                 final TransferStatus status,
                                 final ConnectionCallback callback); @Override Upload<BaseB2Response> withWriter(final Write<BaseB2Response> writer); }
B2LargeUploadService extends HttpUploadFeature<BaseB2Response, MessageDigest> { @Override public BaseB2Response upload(final Path file, final Local local, final BandwidthThrottle throttle, final StreamListener listener, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } if(status.isAppend()) { final B2LargeUploadPartService partService = new B2LargeUploadPartService(session, fileid); final List<B2FileInfoResponse> uploads = partService.find(file); if(uploads.isEmpty()) { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } else { status.setVersion(new VersionId(uploads.iterator().next().getFileId())); completed.addAll(partService.list(status.getVersion().id)); } } else { status.setVersion(new VersionId(session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(file), new DisabledListProgressListener()), containerService.getKey(file), status.getMime(), fileinfo).getFileId())); } final long size = status.getLength() + status.getOffset(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; for(int partNumber = 1; remaining > 0; partNumber++) { boolean skip = false; if(status.isAppend()) { if(log.isInfoEnabled()) { log.info(String.format("Determine if part number %d can be skipped", partNumber)); } for(B2UploadPartResponse c : completed) { if(c.getPartNumber().equals(partNumber)) { if(log.isInfoEnabled()) { log.info(String.format("Skip completed part number %d", partNumber)); } skip = true; offset += c.getContentLength(); break; } } } if(!skip) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, file, local, throttle, listener, status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } final B2FinishLargeFileResponse response = session.getClient().finishLargeFileUpload(status.getVersion().id, checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", file, completed.size())); } status.setComplete(); return response; } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Upload {0} failed", e, file); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map("Upload {0} failed", e, file); } finally { pool.shutdown(false); } } B2LargeUploadService(final B2Session session, final B2FileidProvider fileid, final Write<BaseB2Response> writer);  B2LargeUploadService(final B2Session session, final B2FileidProvider fileid, final Write<BaseB2Response> writer, final Long partSize, final Integer concurrency); @Override BaseB2Response upload(final Path file, final Local local,
                                 final BandwidthThrottle throttle,
                                 final StreamListener listener,
                                 final TransferStatus status,
                                 final ConnectionCallback callback); @Override Upload<BaseB2Response> withWriter(final Write<BaseB2Response> writer); static final int MAXIMUM_UPLOAD_PARTS; static final String X_BZ_INFO_LARGE_FILE_SHA1; }