@Test public void testPreserveOnDirectoryDownwardRecursion() throws IOException { FileSystem fs = FileSystem.get(config); EnumSet<FileAttribute> attributes = EnumSet.allOf(FileAttribute.class); attributes.remove(FileAttribute.ACL); Path src = new Path("/tmp/src2"); Path f0 = new Path("/f0"); Path f1 = new Path("/d1/f1"); Path f2 = new Path("/d1/d2/f2"); Path d1 = new Path("/d1/"); Path d2 = new Path("/d1/d2/"); Path root = new Path("/"); createFile(fs, src); createFile(fs, f0); createFile(fs, f1); createFile(fs, f2); fs.setPermission(src, almostFullPerm); fs.setOwner(src, "somebody", "somebody-group"); fs.setTimes(src, 0, 0); fs.setReplication(src, (short) 1); fs.setPermission(root, fullPerm); fs.setOwner(root, "anybody", "anybody-group"); fs.setTimes(root, 400, 400); fs.setReplication(root, (short) 3); fs.setPermission(d1, fullPerm); fs.setOwner(d1, "anybody", "anybody-group"); fs.setTimes(d1, 400, 400); fs.setReplication(d1, (short) 3); fs.setPermission(d2, fullPerm); fs.setOwner(d2, "anybody", "anybody-group"); fs.setTimes(d2, 300, 300); fs.setReplication(d2, (short) 3); fs.setPermission(f0, fullPerm); fs.setOwner(f0, "anybody", "anybody-group"); fs.setTimes(f0, 200, 200); fs.setReplication(f0, (short) 3); fs.setPermission(f1, fullPerm); fs.setOwner(f1, "anybody", "anybody-group"); fs.setTimes(f1, 200, 200); fs.setReplication(f1, (short) 3); fs.setPermission(f2, fullPerm); fs.setOwner(f2, "anybody", "anybody-group"); fs.setTimes(f2, 200, 200); fs.setReplication(f2, (short) 3); CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); DistCpUtils.preserve(fs, root, srcStatus, attributes, false); cluster.triggerHeartbeats(); CopyListingFileStatus rootStatus = new CopyListingFileStatus(fs.getFileStatus(root)); Assert.assertTrue(srcStatus.getPermission().equals(rootStatus.getPermission())); Assert.assertTrue(srcStatus.getOwner().equals(rootStatus.getOwner())); Assert.assertTrue(srcStatus.getGroup().equals(rootStatus.getGroup())); Assert.assertTrue(srcStatus.getAccessTime() == rootStatus.getAccessTime()); Assert.assertTrue(srcStatus.getModificationTime() == rootStatus.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != rootStatus.getReplication()); CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication()); CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == d2Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == d2Status.getModificationTime()); Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication()); CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); }
public static void preserve(FileSystem targetFS, Path path, CopyListingFileStatus srcFileStatus, EnumSet<FileAttribute> attributes, boolean preserveRawXattrs) throws IOException { FileStatus targetFileStatus = targetFS.getFileStatus(path); String group = targetFileStatus.getGroup(); String user = targetFileStatus.getOwner(); boolean chown = false; if (attributes.contains(FileAttribute.ACL)) { List<AclEntry> srcAcl = srcFileStatus.getAclEntries(); List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus); if (!srcAcl.equals(targetAcl)) { targetFS.setAcl(path, srcAcl); } if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); } } else if (attributes.contains(FileAttribute.PERMISSION) && !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); } final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR); if (preserveXAttrs || preserveRawXattrs) { final String rawNS = XAttr.NameSpace.RAW.name().toLowerCase(); Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) { String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } if (chown) { targetFS.setOwner(path, user, group); } if (attributes.contains(FileAttribute.TIMES)) { targetFS.setTimes(path, srcFileStatus.getModificationTime(), srcFileStatus.getAccessTime()); } }
DistCpUtils { public static void preserve(FileSystem targetFS, Path path, CopyListingFileStatus srcFileStatus, EnumSet<FileAttribute> attributes, boolean preserveRawXattrs) throws IOException { FileStatus targetFileStatus = targetFS.getFileStatus(path); String group = targetFileStatus.getGroup(); String user = targetFileStatus.getOwner(); boolean chown = false; if (attributes.contains(FileAttribute.ACL)) { List<AclEntry> srcAcl = srcFileStatus.getAclEntries(); List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus); if (!srcAcl.equals(targetAcl)) { targetFS.setAcl(path, srcAcl); } if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); } } else if (attributes.contains(FileAttribute.PERMISSION) && !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); } final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR); if (preserveXAttrs || preserveRawXattrs) { final String rawNS = XAttr.NameSpace.RAW.name().toLowerCase(); Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) { String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } if (chown) { targetFS.setOwner(path, user, group); } if (attributes.contains(FileAttribute.TIMES)) { targetFS.setTimes(path, srcFileStatus.getModificationTime(), srcFileStatus.getAccessTime()); } } }
DistCpUtils { public static void preserve(FileSystem targetFS, Path path, CopyListingFileStatus srcFileStatus, EnumSet<FileAttribute> attributes, boolean preserveRawXattrs) throws IOException { FileStatus targetFileStatus = targetFS.getFileStatus(path); String group = targetFileStatus.getGroup(); String user = targetFileStatus.getOwner(); boolean chown = false; if (attributes.contains(FileAttribute.ACL)) { List<AclEntry> srcAcl = srcFileStatus.getAclEntries(); List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus); if (!srcAcl.equals(targetAcl)) { targetFS.setAcl(path, srcAcl); } if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); } } else if (attributes.contains(FileAttribute.PERMISSION) && !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); } final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR); if (preserveXAttrs || preserveRawXattrs) { final String rawNS = XAttr.NameSpace.RAW.name().toLowerCase(); Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) { String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } if (chown) { targetFS.setOwner(path, user, group); } if (attributes.contains(FileAttribute.TIMES)) { targetFS.setTimes(path, srcFileStatus.getModificationTime(), srcFileStatus.getAccessTime()); } }  }
DistCpUtils { public static void preserve(FileSystem targetFS, Path path, CopyListingFileStatus srcFileStatus, EnumSet<FileAttribute> attributes, boolean preserveRawXattrs) throws IOException { FileStatus targetFileStatus = targetFS.getFileStatus(path); String group = targetFileStatus.getGroup(); String user = targetFileStatus.getOwner(); boolean chown = false; if (attributes.contains(FileAttribute.ACL)) { List<AclEntry> srcAcl = srcFileStatus.getAclEntries(); List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus); if (!srcAcl.equals(targetAcl)) { targetFS.setAcl(path, srcAcl); } if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); } } else if (attributes.contains(FileAttribute.PERMISSION) && !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); } final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR); if (preserveXAttrs || preserveRawXattrs) { final String rawNS = XAttr.NameSpace.RAW.name().toLowerCase(); Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) { String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } if (chown) { targetFS.setOwner(path, user, group); } if (attributes.contains(FileAttribute.TIMES)) { targetFS.setTimes(path, srcFileStatus.getModificationTime(), srcFileStatus.getAccessTime()); } }  static long getFileSize(Path path, Configuration configuration); static void publish(Configuration configuration,
                                 String label, T value); static int getInt(Configuration configuration, String label); static long getLong(Configuration configuration, String label); static Class<? extends InputFormat> getStrategy(Configuration conf,
                                                                 DistCpOptions options); static String getRelativePath(Path sourceRootPath, Path childPath); static String packAttributes(EnumSet<FileAttribute> attributes); static EnumSet<FileAttribute> unpackAttributes(String attributes); static void preserve(FileSystem targetFS, Path path,
                              CopyListingFileStatus srcFileStatus,
                              EnumSet<FileAttribute> attributes,
                              boolean preserveRawXattrs); static List<AclEntry> getAcl(FileSystem fileSystem,
      FileStatus fileStatus); static Map<String, byte[]> getXAttrs(FileSystem fileSystem,
      Path path); static CopyListingFileStatus toCopyListingFileStatus(
      FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, 
      boolean preserveXAttrs, boolean preserveRawXAttrs); static Path sortListing(FileSystem fs, Configuration conf, Path sourceListing); static void checkFileSystemAclSupport(FileSystem fs); static void checkFileSystemXAttrSupport(FileSystem fs); static DecimalFormat getFormatter(); static String getStringDescriptionFor(long nBytes); static boolean checksumsAreEqual(FileSystem sourceFS, Path source,
      FileChecksum sourceChecksum, FileSystem targetFS, Path target); static boolean compareFs(FileSystem srcFs, FileSystem destFs); }
DistCpUtils { public static void preserve(FileSystem targetFS, Path path, CopyListingFileStatus srcFileStatus, EnumSet<FileAttribute> attributes, boolean preserveRawXattrs) throws IOException { FileStatus targetFileStatus = targetFS.getFileStatus(path); String group = targetFileStatus.getGroup(); String user = targetFileStatus.getOwner(); boolean chown = false; if (attributes.contains(FileAttribute.ACL)) { List<AclEntry> srcAcl = srcFileStatus.getAclEntries(); List<AclEntry> targetAcl = getAcl(targetFS, targetFileStatus); if (!srcAcl.equals(targetAcl)) { targetFS.setAcl(path, srcAcl); } if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); } } else if (attributes.contains(FileAttribute.PERMISSION) && !srcFileStatus.getPermission().equals(targetFileStatus.getPermission())) { targetFS.setPermission(path, srcFileStatus.getPermission()); } final boolean preserveXAttrs = attributes.contains(FileAttribute.XATTR); if (preserveXAttrs || preserveRawXattrs) { final String rawNS = XAttr.NameSpace.RAW.name().toLowerCase(); Map<String, byte[]> srcXAttrs = srcFileStatus.getXAttrs(); Map<String, byte[]> targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { for (Entry<String, byte[]> entry : srcXAttrs.entrySet()) { String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } if (chown) { targetFS.setOwner(path, user, group); } if (attributes.contains(FileAttribute.TIMES)) { targetFS.setTimes(path, srcFileStatus.getModificationTime(), srcFileStatus.getAccessTime()); } }  static long getFileSize(Path path, Configuration configuration); static void publish(Configuration configuration,
                                 String label, T value); static int getInt(Configuration configuration, String label); static long getLong(Configuration configuration, String label); static Class<? extends InputFormat> getStrategy(Configuration conf,
                                                                 DistCpOptions options); static String getRelativePath(Path sourceRootPath, Path childPath); static String packAttributes(EnumSet<FileAttribute> attributes); static EnumSet<FileAttribute> unpackAttributes(String attributes); static void preserve(FileSystem targetFS, Path path,
                              CopyListingFileStatus srcFileStatus,
                              EnumSet<FileAttribute> attributes,
                              boolean preserveRawXattrs); static List<AclEntry> getAcl(FileSystem fileSystem,
      FileStatus fileStatus); static Map<String, byte[]> getXAttrs(FileSystem fileSystem,
      Path path); static CopyListingFileStatus toCopyListingFileStatus(
      FileSystem fileSystem, FileStatus fileStatus, boolean preserveAcls, 
      boolean preserveXAttrs, boolean preserveRawXAttrs); static Path sortListing(FileSystem fs, Configuration conf, Path sourceListing); static void checkFileSystemAclSupport(FileSystem fs); static void checkFileSystemXAttrSupport(FileSystem fs); static DecimalFormat getFormatter(); static String getStringDescriptionFor(long nBytes); static boolean checksumsAreEqual(FileSystem sourceFS, Path source,
      FileChecksum sourceChecksum, FileSystem targetFS, Path target); static boolean compareFs(FileSystem srcFs, FileSystem destFs);  }