CopyListing extends Configured { public final void buildListing(Path pathToListFile, DistCpContext distCpContext) throws IOException { validatePaths(distCpContext); doBuildListing(pathToListFile, distCpContext); Configuration config = getConf(); config.set(DistCpConstants.CONF_LABEL_LISTING_FILE_PATH, pathToListFile.toString()); config.setLong(DistCpConstants.CONF_LABEL_TOTAL_BYTES_TO_BE_COPIED, getBytesToCopy()); config.setLong(DistCpConstants.CONF_LABEL_TOTAL_NUMBER_OF_RECORDS, getNumberOfPaths()); validateFinalListing(pathToListFile, distCpContext); LOG.info("Number of paths in the copy list: " + this.getNumberOfPaths()); } protected  CopyListing(Configuration configuration, Credentials credentials); final void buildListing(Path pathToListFile,
      DistCpContext distCpContext); static CopyListing getCopyListing(Configuration configuration,
      Credentials credentials, DistCpContext context);  }
@Test(timeout=10000) public void testBuildListing() { FileSystem fs = null; try { fs = FileSystem.get(getConf()); List<Path> srcPaths = new ArrayList<Path>(); Path p1 = new Path("/tmp/in/1"); Path p2 = new Path("/tmp/in/2"); Path p3 = new Path("/tmp/in2/2"); Path target = new Path("/tmp/out/1"); srcPaths.add(p1.getParent()); srcPaths.add(p3.getParent()); TestDistCpUtils.createFile(fs, "/tmp/in/1"); TestDistCpUtils.createFile(fs, "/tmp/in/2"); TestDistCpUtils.createFile(fs, "/tmp/in2/2"); fs.mkdirs(target); OutputStream out = fs.create(p1); out.write("ABC".getBytes()); out.close(); out = fs.create(p2); out.write("DEF".getBytes()); out.close(); out = fs.create(p3); out.write("GHIJ".getBytes()); out.close(); Path listingFile = new Path("/tmp/file"); final DistCpOptions options = new DistCpOptions.Builder(srcPaths, target) .withSyncFolder(true) .build(); CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); try { listing.buildListing(listingFile, new DistCpContext(options)); Assert.fail("Duplicates not detected"); } catch (DuplicateFileException ignore) { } assertThat(listing.getBytesToCopy()).isEqualTo(10); assertThat(listing.getNumberOfPaths()).isEqualTo(3); TestDistCpUtils.delete(fs, "/tmp"); try { listing.buildListing(listingFile, new DistCpContext(options)); Assert.fail("Invalid input not detected"); } catch (InvalidInputException ignore) { } TestDistCpUtils.delete(fs, "/tmp"); } catch (IOException e) { LOG.error("Exception encountered ", e); Assert.fail("Test build listing failed"); } finally { TestDistCpUtils.delete(fs, "/tmp"); } }
@Test(timeout=10000) public void testBuildListingForSingleFile() { FileSystem fs = null; String testRootString = "/singleFileListing"; Path testRoot = new Path(testRootString); SequenceFile.Reader reader = null; try { fs = FileSystem.get(getConf()); if (fs.exists(testRoot)) TestDistCpUtils.delete(fs, testRootString); Path sourceFile = new Path(testRoot, "/source/foo/bar/source.txt"); Path decoyFile = new Path(testRoot, "/target/moo/source.txt"); Path targetFile = new Path(testRoot, "/target/moo/target.txt"); TestDistCpUtils.createFile(fs, sourceFile.toString()); TestDistCpUtils.createFile(fs, decoyFile.toString()); TestDistCpUtils.createFile(fs, targetFile.toString()); List<Path> srcPaths = new ArrayList<Path>(); srcPaths.add(sourceFile); DistCpOptions options = new DistCpOptions.Builder(srcPaths, targetFile) .build(); CopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS); final Path listFile = new Path(testRoot, "/tmp/fileList.seq"); listing.buildListing(listFile, new DistCpContext(options)); reader = new SequenceFile.Reader(getConf(), SequenceFile.Reader.file(listFile)); CopyListingFileStatus fileStatus = new CopyListingFileStatus(); Text relativePath = new Text(); Assert.assertTrue(reader.next(relativePath, fileStatus)); Assert.assertTrue(relativePath.toString().equals("")); } catch (Exception e) { Assert.fail("Unexpected exception encountered."); LOG.error("Unexpected exception: ", e); } finally { TestDistCpUtils.delete(fs, testRootString); IOUtils.closeStream(reader); } }