@Test public void testCopyWithDeleteTargetAndDeleteParentDirectories() throws Exception { Path source = new Path("/source"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1")); List<FileStatus> targetFiles = Lists.newArrayList(createFileStatus(target, "file3")); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); properties.setProperty(RecursiveCopyableDataset.DELETE_EMPTY_DIRECTORIES_KEY, "true"); properties.setProperty(RecursiveCopyableDataset.DELETE_KEY, "true"); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, target, sourceFiles, targetFiles, properties); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.getLocal(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 2); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "file1")); Assert.assertEquals(classifiedFiles.getPathsToDelete().size(), 1); Assert.assertTrue(classifiedFiles.getPathsToDelete().contains(new Path(target, "file3"))); CommitStepCopyEntity entity = (CommitStepCopyEntity) Iterables.filter(copyableFiles, new Predicate<CopyEntity>() { @Override public boolean apply(@Nullable CopyEntity copyEntity) { return copyEntity instanceof CommitStepCopyEntity; } }).iterator().next(); DeleteFileCommitStep step = (DeleteFileCommitStep) entity.getStep(); Assert.assertTrue(step.getParentDeletionLimit().isPresent()); Assert.assertEquals(step.getParentDeletionLimit().get(), target); }
@Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } RecursiveCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, Path glob); }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } RecursiveCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, Path glob); @Override Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration); @Override Path datasetRoot(); @Override String datasetURN(); }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } RecursiveCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, Path glob); @Override Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration); @Override Path datasetRoot(); @Override String datasetURN(); static final String UPDATE_KEY; static final String DELETE_KEY; static final String DELETE_EMPTY_DIRECTORIES_KEY; }