@Test public void testCorrectComputationOfTargetPathsWhenUsingGlob() throws Exception { Path source = new Path("/source/directory"); Path target = new Path("/target"); List<FileStatus> sourceFiles = Lists.newArrayList(createFileStatus(source, "file1")); List<FileStatus> targetFiles = Lists.newArrayList(); Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.DATA_PUBLISHER_FINAL_DIR, target.toString()); FileSystem sourceUnderlyingFS = FileSystem.getLocal(new Configuration()); FileSystem sourceFs = Mockito.spy(sourceUnderlyingFS); Mockito.doReturn(new FileStatus(0, true, 0, 0, 0, source)).when(sourceFs).getFileStatus(source); RecursiveCopyableDataset dataset = new TestRecursiveCopyableDataset(source, new Path(target, "directory"), sourceFiles, targetFiles, properties, new Path("/source/*"), sourceFs); Collection<? extends CopyEntity> copyableFiles = dataset.getCopyableFiles(FileSystem.get(new Configuration()), CopyConfiguration.builder(FileSystem.getLocal(new Configuration()), properties).build()); Assert.assertEquals(copyableFiles.size(), 1); ClassifiedFiles classifiedFiles = classifyFiles(copyableFiles); Assert.assertTrue(classifiedFiles.getPathsToCopy().containsKey(new Path(source, "file1"))); Assert.assertEquals(classifiedFiles.getPathsToCopy().get(new Path(source, "file1")), new Path(target, "directory/file1")); }
@Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } RecursiveCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, Path glob); }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } RecursiveCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, Path glob); @Override Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration); @Override Path datasetRoot(); @Override String datasetURN(); }
RecursiveCopyableDataset implements CopyableDataset, FileSystemDataset { @Override public Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration) throws IOException { Path nonGlobSearchPath = PathUtils.deepestNonGlobPath(this.glob); Path targetPath = new Path(configuration.getPublishDir(), PathUtils.relativizePath(this.rootPath, nonGlobSearchPath)); Map<Path, FileStatus> filesInSource = createPathMap(getFilesAtPath(this.fs, this.rootPath, this.pathFilter), this.rootPath); Map<Path, FileStatus> filesInTarget = createPathMap(getFilesAtPath(targetFs, targetPath, this.pathFilter), targetPath); return getCopyableFilesImpl(configuration, filesInSource, filesInTarget, targetFs, nonGlobSearchPath, configuration.getPublishDir(), targetPath); } RecursiveCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, Path glob); @Override Collection<? extends CopyEntity> getCopyableFiles(FileSystem targetFs, CopyConfiguration configuration); @Override Path datasetRoot(); @Override String datasetURN(); static final String UPDATE_KEY; static final String DELETE_KEY; static final String DELETE_EMPTY_DIRECTORIES_KEY; }