AbstractHadoopGraphComputer implements GraphComputer { public static File copyDirectoryIfNonExistent(final FileSystem fileSystem, final String directory) { try { final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs"; final Path path = new Path(directory); if (Boolean.valueOf(System.getProperty("is.testing", "false")) || (fileSystem.exists(path) && fileSystem.isDirectory(path))) { final File tempDirectory = new File(System.getProperty("java.io.tmpdir"), hadoopGremlinLibsRemote); assert tempDirectory.exists() || tempDirectory.mkdirs(); final Path tempPath = new Path(new Path(tempDirectory.toURI()), path.getName()); final RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(path, false); while (files.hasNext()) { final LocatedFileStatus f = files.next(); fileSystem.copyToLocalFile(false, f.getPath(), new Path(tempPath, f.getPath().getName()), true); } return new File(tempPath.toUri()); } else return new File(directory); } catch (final IOException e) { throw new IllegalStateException(e.getMessage(), e); } } AbstractHadoopGraphComputer(final HadoopGraph hadoopGraph); @Override GraphComputer vertices(final Traversal<Vertex, Vertex> vertexFilter); @Override GraphComputer edges(final Traversal<Vertex, Edge> edgeFilter); @Override GraphComputer result(final ResultGraph resultGraph); @Override GraphComputer persist(final Persist persist); @Override GraphComputer program(final VertexProgram vertexProgram); @Override GraphComputer mapReduce(final MapReduce mapReduce); @Override GraphComputer workers(final int workers); @Override String toString(); @Override Features features(); static File copyDirectoryIfNonExistent(final FileSystem fileSystem, final String directory);  }
@Test public void shouldCopyDirectoriesCorrectly() throws Exception { final String hdfsName = this.getClass().getSimpleName() + "-hdfs"; final String localName = this.getClass().getSimpleName() + "-local"; final FileSystem fs = FileSystem.get(new Configuration()); File path; path = new File(new File(System.getProperty("java.io.tmpdir")), localName); if (!path.exists()) { assertTrue(path.mkdir()); } File tempFile1 = new File(path, "test1.txt"); tempFile1.delete(); assertTrue("Expected "+tempFile1+" file was created successfully", tempFile1.createNewFile()); assertTrue("Expected "+tempFile1+" existed", tempFile1.exists()); File tempFile2 = new File(path, "test2.txt"); tempFile2.delete(); assertTrue("Expected "+tempFile2+" file was created successfully", tempFile2.createNewFile()); assertTrue("Expected "+tempFile2+" existed", tempFile2.exists()); if (fs.exists(new Path("target", hdfsName))) assertTrue(fs.delete(new Path("target", hdfsName), true)); fs.copyFromLocalFile(true, new Path(tempFile1.toURI()), new Path("target", hdfsName + "/test1.dat")); fs.copyFromLocalFile(true, new Path(tempFile2.toURI()), new Path("target", hdfsName + "/test2.dat")); assertTrue(fs.exists(new Path("target", hdfsName + "/test1.dat"))); assertTrue(fs.exists(new Path("target", hdfsName + "/test2.dat"))); assertTrue(fs.exists(new Path("target", hdfsName))); assertTrue(fs.isDirectory(new Path("target", hdfsName))); assertFalse(tempFile1.exists()); assertFalse(tempFile2.exists()); assertTrue("Expected "+path+" directory existed", path.exists()); assertTrue("Expected "+path+" directory was deleted", path.delete()); assertTrue(fs.exists(new Path("target", hdfsName + "/test1.dat"))); assertTrue(fs.exists(new Path("target", hdfsName + "/test2.dat"))); assertTrue(fs.exists(new Path("target",hdfsName))); assertTrue(fs.isDirectory(new Path("target", hdfsName))); final String hadoopGremlinLibsRemote = "hadoop-gremlin-" + Gremlin.version() + "-libs"; final File localDirectory = new File(System.getProperty("java.io.tmpdir"), hadoopGremlinLibsRemote); final File localLibDirectory = new File(localDirectory, hdfsName); if (localLibDirectory.exists()) { Stream.of(localLibDirectory.listFiles()).forEach(File::delete); assertTrue(localLibDirectory.delete()); } assertFalse(localLibDirectory.exists()); assertEquals(localLibDirectory, AbstractHadoopGraphComputer.copyDirectoryIfNonExistent(fs, "target/" + hdfsName)); assertTrue(localLibDirectory.exists()); assertTrue(localLibDirectory.isDirectory()); assertEquals(2, Stream.of(localLibDirectory.listFiles()).filter(file -> file.getName().endsWith(".dat")).count()); }