DistCpUtils { public static void preserve(FileSystem targetFS, Path path, CopyListingFileStatus srcFileStatus, EnumSet<FileAttribute> attributes, boolean preserveRawXattrs) throws IOException { attributes.remove(FileAttribute.BLOCKSIZE); attributes.remove(FileAttribute.CHECKSUMTYPE); FileStatus targetFileStatus = attributes.isEmpty() ? null : targetFS.getFileStatus(path); String group = targetFileStatus == null ? null : targetFileStatus.getGroup(); String user = targetFileStatus == null ? null : targetFileStatus.getOwner(); boolean chown = false; if (attributes.contains(FileAttribute.ACL)) { List<AclEntry> srcAcl = srcFileStatus.getAclEntries(); List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus); if (!srcAcl.equals(targetAcl)) { targetFS.removeAcl(path); targetFS.setAcl(path, srcAcl); } if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); } } else if (attributes.contains(FileAttribute.PERMISSION) && !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); } final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR); if (preserveXAttrs || preserveRawXattrs) { final String rawNS = StringUtils.toLowerCase(XAttr.NameSpace.RAW.name()); Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) { String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && !targetFileStatus.isErasureCoded() && !srcFileStatus.isErasureCoded() && srcFileStatus.getReplication() != targetFileStatus.getReplication()) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } if (chown) { targetFS.setOwner(path, user, group); } if (attributes.contains(FileAttribute.TIMES)) { targetFS.setTimes(path, srcFileStatus.getModificationTime(), srcFileStatus.getAccessTime()); } }  static long getFileSize(Path path, Configuration configuration); static void publish(Configuration configuration,
                                 String label, T value); static int getInt(Configuration configuration, String label); static long getLong(Configuration configuration, String label); static Class<? extends InputFormat> getStrategy(Configuration conf,
      DistCpContext context); static String getRelativePath(Path sourceRootPath, Path childPath); static String packAttributes(EnumSet<FileAttribute> attributes); static EnumSet<FileAttribute> unpackAttributes(String attributes); static void preserve(FileSystem targetFS, Path path,
                              CopyListingFileStatus srcFileStatus,
                              EnumSet<FileAttribute> attributes,
                              boolean preserveRawXattrs); static List<AclEntry> getAcl(FileSystem fileSystem,
      FileStatus fileStatus); static Map<String, byte[]> getXAttrs(FileSystem fileSystem,
      Path path); static LinkedList<CopyListingFileStatus> toCopyListingFileStatus(
      FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls,
      boolean preserveXAttrs, boolean preserveRawXAttrs, int blocksPerChunk); static CopyListingFileStatus toCopyListingFileStatusHelper(
      FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, 
      boolean preserveXAttrs, boolean preserveRawXAttrs,
      long chunkOffset, long chunkLength); static Path sortListing(Configuration conf,
      Path sourceListing); static void sortListing(final Configuration conf,
      final Path sourceListing,
      final Path output); static void checkFileSystemAclSupport(FileSystem fs); static void checkFileSystemXAttrSupport(FileSystem fs); static DecimalFormat getFormatter(); static String getStringDescriptionFor(long nBytes); static boolean checksumsAreEqual(FileSystem sourceFS, Path source,
                                          FileChecksum sourceChecksum,
                                          FileSystem targetFS,
                                          Path target, long sourceLen); static void compareFileLengthsAndChecksums(long srcLen,
             FileSystem sourceFS, Path source, FileChecksum sourceChecksum,
             FileSystem targetFS, Path target, boolean skipCrc,
             long targetLen); static Path getSplitChunkPath(Path targetFile,
      CopyListingFileStatus srcFileStatus);  }
@Test public void testSkipsNeedlessAttributes() throws Exception { FileSystem fs = FileSystem.get(config); Path src = new Path("/tmp/testSkipsNeedlessAttributes/source"); Path dst = new Path("/tmp/testSkipsNeedlessAttributes/dest"); CopyListingFileStatus srcStatus = new CopyListingFileStatus( new FileStatus(0, false, 1, 32, 0, src)); EnumSet<FileAttribute> attrs = EnumSet.of(FileAttribute.ACL, FileAttribute.GROUP, FileAttribute.PERMISSION, FileAttribute.TIMES, FileAttribute.XATTR); for (FileAttribute attr : attrs) { intercept(FileNotFoundException.class, () -> DistCpUtils.preserve(fs, dst, srcStatus, EnumSet.of(attr), false)); } DistCpUtils.preserve(fs, dst, srcStatus, EnumSet.of( FileAttribute.BLOCKSIZE, FileAttribute.CHECKSUMTYPE), false); }
@Test public void testPreserveAclsforDefaultACL() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.ACL, FileAttribute.PERMISSION, FileAttribute.XATTR, FileAttribute.GROUP, FileAttribute.USER, FileAttribute.REPLICATION, FileAttribute.XATTR, FileAttribute.TIMES); Path dest = new Path("/tmpdest"); Path src = new Path("/testsrc"); fs.mkdirs(src); fs.mkdirs(dest); List<AclEntry> acls = Lists.newArrayList( aclEntry(DEFAULT, USER, "foo", READ_EXECUTE), aclEntry(ACCESS, USER, READ_WRITE), aclEntry(ACCESS, GROUP, READ), aclEntry(ACCESS, OTHER, READ), aclEntry(ACCESS, USER, "bar", ALL)); final List<AclEntry> acls1 = Lists.newArrayList(aclEntry(ACCESS, USER, ALL), aclEntry(ACCESS, USER, "user1", ALL), aclEntry(ACCESS, GROUP, READ_EXECUTE), aclEntry(ACCESS, OTHER, EXECUTE)); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setAcl(src, acls); fs.setPermission(dest, noPerm); fs.setOwner(dest, "nobody", "nobody-group"); fs.setTimes(dest, 100, 100); fs.setReplication(dest, (short) 2); fs.setAcl(dest, acls1); List<AclEntry> en1 = fs.getAclStatus(src).getEntries(); List<AclEntry> dd2 = fs.getAclStatus(dest).getEntries(); Assert.assertNotEquals(en1, dd2); CopyListingFileStatus srcStatus = new CopyListingFileStatus( fs.getFileStatus(src)); en1 = srcStatus.getAclEntries(); DistCpUtils.preserve(fs, dest, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus( fs.getFileStatus(dest)); dd2 = dstStatus.getAclEntries(); en1 = srcStatus.getAclEntries(); assertStatusEqual(fs, dest, srcStatus); Assert.assertArrayEquals(en1.toArray(), dd2.toArray()); }
@Test public void testPreserveNothingOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertTrue(dstStatus.getAccessTime() == 100); Assert.assertTrue(dstStatus.getModificationTime() == 100); Assert.assertTrue(dstStatus.getReplication() == 0); }
@Test public void testPreservePermissionOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); }
@Test public void testPreserveGroupOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); }
@Test public void testPreserveUserOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); }
@Test public void testPreserveReplicationOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setReplication(src, (short) 1); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setReplication(dst, (short) 2); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); }
@Test public void testPreserveTimestampOnDirectory() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES); Path dst = new Path("/tmp/abc"); Path src = new Path("/tmp/src"); createDirectory(fs, src); createDirectory(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); }
@Test public void testPreserveNothingOnFile() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.noneOf(FileAttribute.class); Path dst = new Path("/tmp/dest2"); Path src = new Path("/tmp/src2"); createFile(fs, src); createFile(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); fs.setReplication(dst, (short) 2); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); assertStatusNotEqual(fs, dst, srcStatus); }
@Test public void testPreservePermissionOnFile() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.PERMISSION); Path dst = new Path("/tmp/dest2"); Path src = new Path("/tmp/src2"); createFile(fs, src); createFile(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); fs.setReplication(dst, (short) 2); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); }
@Test public void testPreserveGroupOnFile() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.GROUP); Path dst = new Path("/tmp/dest2"); Path src = new Path("/tmp/src2"); createFile(fs, src); createFile(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); fs.setReplication(dst, (short) 2); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); }
@Test public void testPreserveUserOnFile() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.USER); Path dst = new Path("/tmp/dest2"); Path src = new Path("/tmp/src2"); createFile(fs, src); createFile(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); fs.setReplication(dst, (short) 2); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); }
@Test public void testPreserveReplicationOnFile() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.REPLICATION); Path dst = new Path("/tmp/dest2"); Path src = new Path("/tmp/src2"); createFile(fs, src); createFile(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); fs.setReplication(dst, (short) 2); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); }
@Test public void testPreserveTimestampOnFile() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.of(FileAttribute.TIMES); Path dst = new Path("/tmp/dest2"); Path src = new Path("/tmp/src2"); createFile(fs, src); createFile(fs, dst); fs.setPermission(src, fullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(dst, noPerm); fs.setOwner(dst, "nobody", "nobody-group"); fs.setTimes(dst, 100, 100); fs.setReplication(dst, (short) 2); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); }
@Test public void testPreserveOnFileUpwardRecursion() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); attributes.remove(FileAttribute.ACL); Path src = new Path("/tmp/src2"); Path f0 = new Path("/f0"); Path f1 = new Path("/d1/f1"); Path f2 = new Path("/d1/d2/f2"); Path d1 = new Path("/d1/"); Path d2 = new Path("/d1/d2/"); createFile(fs, src); createFile(fs, f0); createFile(fs, f1); createFile(fs, f2); fs.setPermission(src, almostFullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(d1, fullPerm); fs.setOwner(d1, "anybody", "anybody-group"); fs.setTimes(d1, 400, 400); fs.setReplication(d1, (short) 3); fs.setPermission(d2, fullPerm); fs.setOwner(d2, "anybody", "anybody-group"); fs.setTimes(d2, 300, 300); fs.setReplication(d2, (short) 3); fs.setPermission(f0, fullPerm); fs.setOwner(f0, "anybody", "anybody-group"); fs.setTimes(f0, 200, 200); fs.setReplication(f0, (short) 3); fs.setPermission(f1, fullPerm); fs.setOwner(f1, "anybody", "anybody-group"); fs.setTimes(f1, 200, 200); fs.setReplication(f1, (short) 3); fs.setPermission(f2, fullPerm); fs.setOwner(f2, "anybody", "anybody-group"); fs.setTimes(f2, 200, 200); fs.setReplication(f2, (short) 3); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, f2, srcStatus, attributes, false); cluster.triggerHeartbeats(); assertStatusEqual(fs, f2, srcStatus); CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); Assert.assertTrue(d2Status.getAccessTime() == 300); Assert.assertTrue(d2Status.getModificationTime() == 300); Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication()); CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); Assert.assertTrue(d1Status.getAccessTime() == 400); Assert.assertTrue(d1Status.getModificationTime() == 400); Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication()); }
@Test public void testPreserveOnDirectoryUpwardRecursion() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); attributes.remove(FileAttribute.ACL); Path src = new Path("/tmp/src2"); Path f0 = new Path("/f0"); Path f1 = new Path("/d1/f1"); Path f2 = new Path("/d1/d2/f2"); Path d1 = new Path("/d1/"); Path d2 = new Path("/d1/d2/"); createFile(fs, src); createFile(fs, f0); createFile(fs, f1); createFile(fs, f2); fs.setPermission(src, almostFullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(d1, fullPerm); fs.setOwner(d1, "anybody", "anybody-group"); fs.setTimes(d1, 400, 400); fs.setReplication(d1, (short) 3); fs.setPermission(d2, fullPerm); fs.setOwner(d2, "anybody", "anybody-group"); fs.setTimes(d2, 300, 300); fs.setReplication(d2, (short) 3); fs.setPermission(f0, fullPerm); fs.setOwner(f0, "anybody", "anybody-group"); fs.setTimes(f0, 200, 200); fs.setReplication(f0, (short) 3); fs.setPermission(f1, fullPerm); fs.setOwner(f1, "anybody", "anybody-group"); fs.setTimes(f1, 200, 200); fs.setReplication(f1, (short) 3); fs.setPermission(f2, fullPerm); fs.setOwner(f2, "anybody", "anybody-group"); fs.setTimes(f2, 200, 200); fs.setReplication(f2, (short) 3); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, d2, srcStatus, attributes, false); cluster.triggerHeartbeats(); CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); Assert.assertTrue(srcStatus.getPermission().equals(d2Status.getPermission())); Assert.assertTrue(srcStatus.getOwner().equals(d2Status.getOwner())); Assert.assertTrue(srcStatus.getGroup().equals(d2Status.getGroup())); Assert.assertTrue(srcStatus.getAccessTime() == d2Status.getAccessTime()); Assert.assertTrue(srcStatus.getModificationTime() == d2Status.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication()); CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication()); CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); }
@Test public void testPreserveOnFileDownwardRecursion() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); attributes.remove(FileAttribute.ACL); Path src = new Path("/tmp/src2"); Path f0 = new Path("/f0"); Path f1 = new Path("/d1/f1"); Path f2 = new Path("/d1/d2/f2"); Path d1 = new Path("/d1/"); Path d2 = new Path("/d1/d2/"); createFile(fs, src); createFile(fs, f0); createFile(fs, f1); createFile(fs, f2); fs.setPermission(src, almostFullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(d1, fullPerm); fs.setOwner(d1, "anybody", "anybody-group"); fs.setTimes(d1, 400, 400); fs.setReplication(d1, (short) 3); fs.setPermission(d2, fullPerm); fs.setOwner(d2, "anybody", "anybody-group"); fs.setTimes(d2, 300, 300); fs.setReplication(d2, (short) 3); fs.setPermission(f0, fullPerm); fs.setOwner(f0, "anybody", "anybody-group"); fs.setTimes(f0, 200, 200); fs.setReplication(f0, (short) 3); fs.setPermission(f1, fullPerm); fs.setOwner(f1, "anybody", "anybody-group"); fs.setTimes(f1, 200, 200); fs.setReplication(f1, (short) 3); fs.setPermission(f2, fullPerm); fs.setOwner(f2, "anybody", "anybody-group"); fs.setTimes(f2, 200, 200); fs.setReplication(f2, (short) 3); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, f0, srcStatus, attributes, false); cluster.triggerHeartbeats(); assertStatusEqual(fs, f0, srcStatus); CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); Assert.assertTrue(d1Status.getAccessTime() == 400); Assert.assertTrue(d1Status.getModificationTime() == 400); Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication()); CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); Assert.assertTrue(d2Status.getAccessTime() == 300); Assert.assertTrue(d2Status.getModificationTime() == 300); Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication()); }
@Test public void testPreserveOnDirectoryDownwardRecursion() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); attributes.remove(FileAttribute.ACL); Path src = new Path("/tmp/src2"); Path f0 = new Path("/f0"); Path f1 = new Path("/d1/f1"); Path f2 = new Path("/d1/d2/f2"); Path d1 = new Path("/d1/"); Path d2 = new Path("/d1/d2/"); Path root = new Path("/"); createFile(fs, src); createFile(fs, f0); createFile(fs, f1); createFile(fs, f2); fs.setPermission(src, almostFullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(root, fullPerm); fs.setOwner(root, "anybody", "anybody-group"); fs.setTimes(root, 400, 400); fs.setReplication(root, (short) 3); fs.setPermission(d1, fullPerm); fs.setOwner(d1, "anybody", "anybody-group"); fs.setTimes(d1, 400, 400); fs.setReplication(d1, (short) 3); fs.setPermission(d2, fullPerm); fs.setOwner(d2, "anybody", "anybody-group"); fs.setTimes(d2, 300, 300); fs.setReplication(d2, (short) 3); fs.setPermission(f0, fullPerm); fs.setOwner(f0, "anybody", "anybody-group"); fs.setTimes(f0, 200, 200); fs.setReplication(f0, (short) 3); fs.setPermission(f1, fullPerm); fs.setOwner(f1, "anybody", "anybody-group"); fs.setTimes(f1, 200, 200); fs.setReplication(f1, (short) 3); fs.setPermission(f2, fullPerm); fs.setOwner(f2, "anybody", "anybody-group"); fs.setTimes(f2, 200, 200); fs.setReplication(f2, (short) 3); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, root, srcStatus, attributes, false); cluster.triggerHeartbeats(); CopyListingFileStatus rootStatus = new CopyListingFileStatus(fs.getFileStatus(root)); Assert.assertTrue(srcStatus.getPermission().equals(rootStatus.getPermission())); Assert.assertTrue(srcStatus.getOwner().equals(rootStatus.getOwner())); Assert.assertTrue(srcStatus.getGroup().equals(rootStatus.getGroup())); Assert.assertTrue(srcStatus.getAccessTime() == rootStatus.getAccessTime()); Assert.assertTrue(srcStatus.getModificationTime() == rootStatus.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != rootStatus.getReplication()); CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication()); CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == d2Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == d2Status.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication()); CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); }