Add option for AccumulatorPathVisitor to ignore file visitation failures
MODIFY
src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
src/main/java/org/apache/commons/io/file/AccumulatorPathVisitor.java
package org.apache.commons.io.file;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Objects;
import org.apache.commons.io.file.Counters.PathCounters;
+import org.apache.commons.io.function.IOBiFunction;
/**
* Accumulates normalized paths during visitation.
@@ -135,6 +137,20 @@ public AccumulatorPathVisitor(final PathCounters pathCounter, final PathFilter f
super(pathCounter, fileFilter, dirFilter);
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param pathCounter How to count path visits.
+ * @param fileFilter Filters which files to count.
+ * @param dirFilter Filters which directories to count.
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public AccumulatorPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter,
+ final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(pathCounter, fileFilter, dirFilter, visitFileFailed);
+ }
+
private void add(final List<Path> list, final Path dir) {
list.add(dir.normalize());
}

MODIFY
src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
src/main/java/org/apache/commons/io/file/CountingPathVisitor.java
import java.util.Objects;
import org.apache.commons.io.file.Counters.PathCounters;
+import org.apache.commons.io.filefilter.IOFileFilter;
import org.apache.commons.io.filefilter.SymbolicLinkFileFilter;
import org.apache.commons.io.filefilter.TrueFileFilter;
+import org.apache.commons.io.function.IOBiFunction;
/**
* Counts files, directories, and sizes, as a visit proceeds.
static final String[] EMPTY_STRING_ARRAY = {};
+ static IOFileFilter defaultDirFilter() {
+ return TrueFileFilter.INSTANCE;
+ }
+
+ static IOFileFilter defaultFileFilter() {
+ return new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE);
+ }
+
/**
* Creates a new instance configured with a {@link BigInteger} {@link PathCounters}.
*
@@ -66,7 +76,7 @@ public static CountingPathVisitor withLongCounters() {
* @param pathCounter How to count path visits.
*/
public CountingPathVisitor(final PathCounters pathCounter) {
- this(pathCounter, new SymbolicLinkFileFilter(FileVisitResult.TERMINATE, FileVisitResult.CONTINUE), TrueFileFilter.INSTANCE);
+ this(pathCounter, defaultFileFilter(), defaultDirFilter());
}
/**
@@ -83,6 +93,23 @@ public CountingPathVisitor(final PathCounters pathCounter, final PathFilter file
this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter");
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param pathCounter How to count path visits.
+ * @param fileFilter Filters which files to count.
+ * @param dirFilter Filters which directories to count.
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public CountingPathVisitor(final PathCounters pathCounter, final PathFilter fileFilter, final PathFilter dirFilter,
+ final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(visitFileFailed);
+ this.pathCounters = Objects.requireNonNull(pathCounter, "pathCounter");
+ this.fileFilter = Objects.requireNonNull(fileFilter, "fileFilter");
+ this.dirFilter = Objects.requireNonNull(dirFilter, "dirFilter");
+ }
+
@Override
public boolean equals(final Object obj) {
if (this == obj) {

MODIFY
src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
src/main/java/org/apache/commons/io/file/NoopPathVisitor.java
package org.apache.commons.io.file;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Path;
+
+import org.apache.commons.io.function.IOBiFunction;
+
/**
* A noop path visitor.
*
* The singleton instance.
*/
public static final NoopPathVisitor INSTANCE = new NoopPathVisitor();
+
+ /**
+ * Constructs a new instance.
+ *
+ * @since 2.12.0
+ */
+ public NoopPathVisitor() {
+ }
+
+ /**
+ * Constructs a new instance.
+ *
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ * @since 2.12.0
+ */
+ public NoopPathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ super(visitFileFailed);
+ }
}

MODIFY
src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
src/main/java/org/apache/commons/io/file/SimplePathVisitor.java
package org.apache.commons.io.file;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
+import java.util.Objects;
+
+import org.apache.commons.io.function.IOBiFunction;
/**
* A {@link SimpleFileVisitor} typed to a {@link Path}.
*/
public abstract class SimplePathVisitor extends SimpleFileVisitor<Path> implements PathVisitor {
+ private final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailedFunction;
+
/**
* Constructs a new instance.
*/
protected SimplePathVisitor() {
+ this.visitFileFailedFunction = super::visitFileFailed;
}
+ /**
+ * Constructs a new instance.
+ *
+ * @param visitFileFailed Called on {@link #visitFileFailed(Path, IOException)}.
+ */
+ protected SimplePathVisitor(final IOBiFunction<Path, IOException, FileVisitResult> visitFileFailed) {
+ this.visitFileFailedFunction = Objects.requireNonNull(visitFileFailed, "visitFileFailed");
+ }
+
+ @Override
+ public FileVisitResult visitFileFailed(final Path file, final IOException exc) throws IOException {
+ return visitFileFailedFunction.apply(file, exc);
+ }
}

MODIFY
src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
src/test/java/org/apache/commons/io/file/AccumulatorPathVisitorTest.java
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.IOException;
+import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Supplier;
import java.util.stream.Stream;
import org.apache.commons.io.filefilter.DirectoryFileFilter;
import org.apache.commons.io.filefilter.EmptyFileFilter;
import org.apache.commons.io.filefilter.PathVisitorFileFilter;
+import org.apache.commons.io.function.IOBiFunction;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.Arguments;
// @formatter:on
}
+ static Stream<Arguments> testParametersIgnoreFailures() {
+ // @formatter:off
+ return Stream.of(
+ Arguments.of((Supplier<AccumulatorPathVisitor>) () -> new AccumulatorPathVisitor(
+ Counters.bigIntegerPathCounters(),
+ CountingPathVisitor.defaultDirFilter(),
+ CountingPathVisitor.defaultFileFilter(),
+ IOBiFunction.noop())));
+ // @formatter:on
+ }
+
@TempDir
Path tempDirPath;
@@ -109,4 +132,97 @@ public void testFolders2FileSize2(final Supplier<AccumulatorPathVisitor> supplie
assertEquals(2, accPathVisitor.getFileList().size());
}
+ /**
+ * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit.
+ *
+ * Random failure like:
+ *
+ * <pre>
+ * ...?...
+ * </pre>
+ */
+ @ParameterizedTest
+ @MethodSource("testParametersIgnoreFailures")
+ public void testFolderWhileDeletingAsync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException, InterruptedException {
+ final int count = 10_000;
+ final List<Path> files = new ArrayList<>(count);
+ // Create "count" file fixtures
+ for (int i = 1; i <= count; i++) {
+ final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt");
+ assertTrue(Files.exists(tempFile));
+ files.add(tempFile);
+ }
+ final AccumulatorPathVisitor accPathVisitor = supplier.get();
+ final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) {
+ @Override
+ public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException {
+ // Slow down the walking a bit to try and cause conflicts with the deletion thread
+ try {
+ Thread.sleep(10);
+ } catch (final InterruptedException ignore) {
+ // e.printStackTrace();
+ }
+ return super.visitFile(path, attributes);
+ }
+ };
+ final ExecutorService executor = Executors.newSingleThreadExecutor();
+ final AtomicBoolean deleted = new AtomicBoolean();
+ try {
+ executor.execute(() -> {
+ for (final Path file : files) {
+ try {
+ // File deletion is slow compared to tree walking, so we go as fast as we can here
+ Files.delete(file);
+ } catch (final IOException ignored) {
+ // e.printStackTrace();
+ }
+ }
+ deleted.set(true);
+ });
+ Files.walkFileTree(tempDirPath, countingFileFilter);
+ } finally {
+ if (!deleted.get()) {
+ Thread.sleep(1000);
+ }
+ if (!deleted.get()) {
+ executor.awaitTermination(5, TimeUnit.SECONDS);
+ }
+ executor.shutdownNow();
+ }
+ }
+
+ /**
+ * Tests IO-755 with a directory with 100 files, and delete all of them mid-way through the visit.
+ */
+ @ParameterizedTest
+ @MethodSource("testParametersIgnoreFailures")
+ public void testFolderWhileDeletingSync(final Supplier<AccumulatorPathVisitor> supplier) throws IOException {
+ final int count = 100;
+ final int marker = count / 2;
+ final Set<Path> files = new LinkedHashSet<>(count);
+ for (int i = 1; i <= count; i++) {
+ final Path tempFile = Files.createTempFile(tempDirPath, "test", ".txt");
+ assertTrue(Files.exists(tempFile));
+ files.add(tempFile);
+ }
+ final AccumulatorPathVisitor accPathVisitor = supplier.get();
+ final AtomicInteger visitCount = new AtomicInteger();
+ final PathVisitorFileFilter countingFileFilter = new PathVisitorFileFilter(accPathVisitor) {
+ @Override
+ public FileVisitResult visitFile(final Path path, final BasicFileAttributes attributes) throws IOException {
+ if (visitCount.incrementAndGet() == marker) {
+ // Now that we've visited half the files, delete them all
+ for (final Path file : files) {
+ Files.delete(file);
+ }
+ }
+ return super.visitFile(path, attributes);
+ }
+ };
+ Files.walkFileTree(tempDirPath, countingFileFilter);
+ assertCounts(1, marker - 1, 0, accPathVisitor.getPathCounters());
+ assertEquals(1, accPathVisitor.getDirList().size());
+ assertEquals(marker - 1, accPathVisitor.getFileList().size());
+ }
+
}

Add option for AccumulatorPathVisitor to ignore file visitation failures