HdfsUtils { public static void collectPaths(Path path, FileSystem fs, PathFilter filter, final List<Path> accumulator) throws IOException { walkPath(path, fs, filter, new PathVisitor() { @Override public void visit(FileStatus fileStatus) { accumulator.add(fileStatus.getPath()); } }); } private  HdfsUtils(); static void walkPath(Path path,
                              FileSystem fs,
                              PathFilter filter,
                              PathVisitor visitor); static void collectPaths(Path path,
                                  FileSystem fs,
                                  PathFilter filter,
                                  final List<Path> accumulator); static long getDirectorySize(Path path, FileSystem fs, PathFilter filter); static long getDirectorySize(Path path, FileSystem fs); static List<Path> expandGlobs(List<String> pathsWithGlobs, Configuration conf);  }
@Test public void testCollectPathsWithDirs() throws Exception { List<Path> accumulator = Lists.newLinkedList(); Path p = new Path(SAMPLE_DIR_LOCATION + "sample_dir"); HdfsUtils.collectPaths( p, p.getFileSystem(new Configuration()), SKIP_A_PATH_FILTER, accumulator); Set<String> expected = Sets.newHashSet( "sample_dir", "sample_dir/b.txt", "sample_dir/nested", "sample_dir/nested/c.txt", "sample_dir/nested/d.txt", "sample_dir/nested/double_nested", "sample_dir/nested/double_nested/e.txt"); Set<String> found = Sets.newHashSet(Iterables.transform(accumulator, PATH_TO_RELATIVE_STRING)); assertEquals(expected, found); }
@Test public void testCollectPathsWithoutDirs() throws Exception { List<Path> accumulator = Lists.newLinkedList(); Configuration conf = new Configuration(); Path p = new Path(SAMPLE_DIR_LOCATION + "sample_dir"); HdfsUtils.collectPaths( p, p.getFileSystem(conf), new PathFilters.CompositePathFilter( PathFilters.newExcludeDirectoriesFilter(conf), SKIP_A_PATH_FILTER), accumulator ); Set<String> expected = Sets.newHashSet( "sample_dir/b.txt", "sample_dir/nested/c.txt", "sample_dir/nested/d.txt", "sample_dir/nested/double_nested/e.txt"); Set<String> found = Sets.newHashSet(Iterables.transform(accumulator, PATH_TO_RELATIVE_STRING)); assertEquals(expected, found); }