@Test public void testCopy() throws Exception { final B2FileidProvider fileid = new B2FileidProvider(session).withCache(cache); final Path container = new Path("test-cyberduck", EnumSet.of(Path.Type.directory, Path.Type.volume)); final String name = new AlphanumericRandomStringService().random(); final byte[] content = RandomUtils.nextBytes(6 * 1024 * 1024); final Path test = new Path(container, name, EnumSet.of(Path.Type.file)); final OutputStream out = new B2WriteFeature(session, fileid).write(test, new TransferStatus().length(content.length), new DisabledConnectionCallback()); new StreamCopier(new TransferStatus(), new TransferStatus().length(content.length)).transfer(new ByteArrayInputStream(content), out); out.close(); assertTrue(new B2FindFeature(session, fileid).find(test)); final Path copy = new B2LargeCopyFeature(session, fileid, 5 * 1024L * 1024L, 1).copy(test, new Path(container, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file)), new TransferStatus().length(content.length), new DisabledConnectionCallback()); assertTrue(new B2FindFeature(session, fileid).find(new Path(container, name, EnumSet.of(Path.Type.file)))); assertTrue(new B2FindFeature(session, fileid).find(copy)); final byte[] compare = new byte[content.length]; final InputStream stream = new B2ReadFeature(session, fileid).read(copy, new TransferStatus().length(content.length), new DisabledConnectionCallback()); IOUtils.readFully(stream, compare); stream.close(); assertArrayEquals(content, compare); new B2DeleteFeature(session, fileid).delete(Arrays.asList(test, copy), new DisabledLoginCallback(), new Delete.DisabledCallback()); session.close(); }
@Override public Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } final B2StartLargeFileResponse response = session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(target), new DisabledListProgressListener()), containerService.getKey(target), status.getMime(), fileinfo); final long size = status.getLength(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); for(int partNumber = 1; remaining > 0; partNumber++) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, source, response.getFileId(), status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } session.getClient().finishLargeFileUpload(response.getFileId(), checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", target, completed.size())); } return new Path(target.getParent(), target.getName(), target.getType(), new PathAttributes(source.attributes()).withVersionId(response.getFileId())); } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Cannot copy {0}", e, source); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map(e); } finally { pool.shutdown(false); } }
B2LargeCopyFeature implements Copy { @Override public Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } final B2StartLargeFileResponse response = session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(target), new DisabledListProgressListener()), containerService.getKey(target), status.getMime(), fileinfo); final long size = status.getLength(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); for(int partNumber = 1; remaining > 0; partNumber++) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, source, response.getFileId(), status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } session.getClient().finishLargeFileUpload(response.getFileId(), checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", target, completed.size())); } return new Path(target.getParent(), target.getName(), target.getType(), new PathAttributes(source.attributes()).withVersionId(response.getFileId())); } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Cannot copy {0}", e, source); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map(e); } finally { pool.shutdown(false); } } }
B2LargeCopyFeature implements Copy { @Override public Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } final B2StartLargeFileResponse response = session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(target), new DisabledListProgressListener()), containerService.getKey(target), status.getMime(), fileinfo); final long size = status.getLength(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); for(int partNumber = 1; remaining > 0; partNumber++) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, source, response.getFileId(), status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } session.getClient().finishLargeFileUpload(response.getFileId(), checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", target, completed.size())); } return new Path(target.getParent(), target.getName(), target.getType(), new PathAttributes(source.attributes()).withVersionId(response.getFileId())); } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Cannot copy {0}", e, source); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map(e); } finally { pool.shutdown(false); } } B2LargeCopyFeature(final B2Session session, final B2FileidProvider fileid);  B2LargeCopyFeature(final B2Session session, final B2FileidProvider fileid, final Long partSize, final Integer concurrency); }
B2LargeCopyFeature implements Copy { @Override public Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } final B2StartLargeFileResponse response = session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(target), new DisabledListProgressListener()), containerService.getKey(target), status.getMime(), fileinfo); final long size = status.getLength(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); for(int partNumber = 1; remaining > 0; partNumber++) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, source, response.getFileId(), status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } session.getClient().finishLargeFileUpload(response.getFileId(), checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", target, completed.size())); } return new Path(target.getParent(), target.getName(), target.getType(), new PathAttributes(source.attributes()).withVersionId(response.getFileId())); } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Cannot copy {0}", e, source); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map(e); } finally { pool.shutdown(false); } } B2LargeCopyFeature(final B2Session session, final B2FileidProvider fileid);  B2LargeCopyFeature(final B2Session session, final B2FileidProvider fileid, final Long partSize, final Integer concurrency); @Override Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback); }
B2LargeCopyFeature implements Copy { @Override public Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException { final ThreadPool pool = ThreadPoolFactory.get("largeupload", concurrency); try { final Map<String, String> fileinfo = new HashMap<>(status.getMetadata()); if(null != status.getTimestamp()) { fileinfo.put(X_BZ_INFO_SRC_LAST_MODIFIED_MILLIS, String.valueOf(status.getTimestamp())); } final Checksum checksum = status.getChecksum(); if(Checksum.NONE != checksum) { switch(checksum.algorithm) { case sha1: fileinfo.put(X_BZ_INFO_LARGE_FILE_SHA1, status.getChecksum().hash); break; } } final B2StartLargeFileResponse response = session.getClient().startLargeFileUpload(fileid.getFileid(containerService.getContainer(target), new DisabledListProgressListener()), containerService.getKey(target), status.getMime(), fileinfo); final long size = status.getLength(); final List<Future<B2UploadPartResponse>> parts = new ArrayList<Future<B2UploadPartResponse>>(); long remaining = status.getLength(); long offset = 0; final List<B2UploadPartResponse> completed = new ArrayList<B2UploadPartResponse>(); for(int partNumber = 1; remaining > 0; partNumber++) { final Long length = Math.min(Math.max((size / B2LargeUploadService.MAXIMUM_UPLOAD_PARTS), partSize), remaining); parts.add(this.submit(pool, source, response.getFileId(), status, partNumber, offset, length, callback)); if(log.isDebugEnabled()) { log.debug(String.format("Part %s submitted with size %d and offset %d", partNumber, length, offset)); } remaining -= length; offset += length; } try { for(Future<B2UploadPartResponse> f : parts) { completed.add(f.get()); } } catch(InterruptedException e) { log.error("Part upload failed with interrupt failure"); status.setCanceled(); throw new ConnectionCanceledException(e); } catch(ExecutionException e) { log.warn(String.format("Part upload failed with execution failure %s", e.getMessage())); if(e.getCause() instanceof BackgroundException) { throw (BackgroundException) e.getCause(); } throw new DefaultExceptionMappingService().map(e.getCause()); } completed.sort(new Comparator<B2UploadPartResponse>() { @Override public int compare(final B2UploadPartResponse o1, final B2UploadPartResponse o2) { return o1.getPartNumber().compareTo(o2.getPartNumber()); } }); final List<String> checksums = new ArrayList<String>(); for(B2UploadPartResponse part : completed) { checksums.add(part.getContentSha1()); } session.getClient().finishLargeFileUpload(response.getFileId(), checksums.toArray(new String[checksums.size()])); if(log.isInfoEnabled()) { log.info(String.format("Finished large file upload %s with %d parts", target, completed.size())); } return new Path(target.getParent(), target.getName(), target.getType(), new PathAttributes(source.attributes()).withVersionId(response.getFileId())); } catch(B2ApiException e) { throw new B2ExceptionMappingService().map("Cannot copy {0}", e, source); } catch(IOException e) { throw new DefaultIOExceptionMappingService().map(e); } finally { pool.shutdown(false); } } B2LargeCopyFeature(final B2Session session, final B2FileidProvider fileid);  B2LargeCopyFeature(final B2Session session, final B2FileidProvider fileid, final Long partSize, final Integer concurrency); @Override Path copy(final Path source, final Path target, final TransferStatus status, final ConnectionCallback callback);  }