diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b2a119f43b3..a4b33140e15 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -543,6 +543,8 @@ Release 2.6.0 - UNRELEASED HADOOP-11017. KMS delegation token secret manager should be able to use zookeeper as store. (asuresh via tucu) + HADOOP-11009. Add Timestamp Preservation to DistCp (Gary Steelman via aw) + OPTIMIZATIONS HADOOP-10838. Byte array native checksumming. (James Thomas via todd) diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java index 04437d78a23..9b31dfbdd8f 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java @@ -101,7 +101,7 @@ public void setAclEntries(List aclEntries) { * @return Map containing all xAttrs */ public Map getXAttrs() { - return xAttrs; + return xAttrs != null ? xAttrs : Collections.emptyMap(); } /** diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java index c5448132678..9b85997ef95 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java @@ -37,18 +37,21 @@ public enum DistCpOptionSwitch { /** * Preserves status of file/path in the target. * Default behavior with -p, is to preserve replication, - * block size, user, group, permission and checksum type on the target file. - * Note that when preserving checksum type, block size is also preserved. + * block size, user, group, permission, checksum type and timestamps on the + * target file. Note that when preserving checksum type, block size is also + * preserved. * - * If any of the optional switches are present among rbugpc, then + * @see PRESERVE_STATUS_DEFAULT + * + * If any of the optional switches are present among rbugpcaxt, then * only the corresponding file attribute is preserved. - * */ PRESERVE_STATUS(DistCpConstants.CONF_LABEL_PRESERVE_STATUS, - new Option("p", true, "preserve status (rbugpcax)(replication, " + - "block-size, user, group, permission, checksum-type, ACL, XATTR). " + - "If -p is specified with no , then preserves replication, " + - "block size, user, group, permission and checksum type." + + new Option("p", true, "preserve status (rbugpcaxt)(replication, " + + "block-size, user, group, permission, checksum-type, ACL, XATTR, " + + "timestamps). If -p is specified with no , then preserves " + + "replication, block size, user, group, permission, checksum type " + + "and timestamps. " + "raw.* xattrs are preserved when both the source and destination " + "paths are in the /.reserved/raw hierarchy (HDFS only). raw.* xattr" + "preservation is independent of the -p flag." + @@ -166,7 +169,7 @@ public enum DistCpOptionSwitch { BANDWIDTH(DistCpConstants.CONF_LABEL_BANDWIDTH_MB, new Option("bandwidth", true, "Specify bandwidth per map in MB")); - static final String PRESERVE_STATUS_DEFAULT = "-prbugpc"; + public static final String PRESERVE_STATUS_DEFAULT = "-prbugpct"; private final String confLabel; private final Option option; diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java index a2a5be3a0b7..57d2fb7eb1c 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java @@ -68,7 +68,7 @@ public class DistCpOptions { private boolean targetPathExists = true; public static enum FileAttribute{ - REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR; + REPLICATION, BLOCKSIZE, USER, GROUP, PERMISSION, CHECKSUMTYPE, ACL, XATTR, TIMES; public static FileAttribute getAttribute(char symbol) { for (FileAttribute attribute : values()) { diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java index abd30eee913..71e84a11cc8 100644 --- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java @@ -18,39 +18,39 @@ package org.apache.hadoop.tools.util; -import com.google.common.collect.Maps; +import java.io.IOException; +import java.net.InetAddress; +import java.net.URI; +import java.net.UnknownHostException; +import java.text.DecimalFormat; +import java.util.EnumSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileChecksum; import org.apache.hadoop.fs.XAttr; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.fs.permission.AclUtil; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.io.SequenceFile; import org.apache.hadoop.io.Text; +import org.apache.hadoop.mapreduce.InputFormat; +import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException; import org.apache.hadoop.tools.CopyListing.XAttrsNotSupportedException; import org.apache.hadoop.tools.CopyListingFileStatus; +import org.apache.hadoop.tools.DistCpOptions; import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.apache.hadoop.tools.mapred.UniformSizeInputFormat; -import org.apache.hadoop.tools.CopyListing.AclsNotSupportedException; -import org.apache.hadoop.tools.DistCpOptions; -import org.apache.hadoop.mapreduce.InputFormat; -import java.io.IOException; -import java.util.EnumSet; -import java.util.Iterator; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Map.Entry; -import java.text.DecimalFormat; -import java.net.URI; -import java.net.InetAddress; -import java.net.UnknownHostException; +import com.google.common.collect.Maps; /** * Utility functions used in DistCp. @@ -163,7 +163,7 @@ public static String packAttributes(EnumSet attributes) { } /** - * Un packs preservation attribute string containing the first character of + * Unpacks preservation attribute string containing the first character of * each preservation attribute back to a set of attributes to preserve * @param attributes - Attribute string * @return - Attribute set @@ -209,7 +209,7 @@ public static void preserve(FileSystem targetFS, Path path, if (!srcAcl.equals(targetAcl)) { targetFS.setAcl(path, srcAcl); } - // setAcl can't preserve sticky bit, so also call setPermission if needed. + // setAcl doesn't preserve sticky bit, so also call setPermission if needed. if (srcFileStatus.getPermission().getStickyBit() != targetFileStatus.getPermission().getStickyBit()) { targetFS.setPermission(path, srcFileStatus.getPermission()); @@ -225,30 +225,28 @@ public static void preserve(FileSystem targetFS, Path path, Map srcXAttrs = srcFileStatus.getXAttrs(); Map targetXAttrs = getXAttrs(targetFS, path); if (srcXAttrs != null && !srcXAttrs.equals(targetXAttrs)) { - Iterator> iter = srcXAttrs.entrySet().iterator(); - while (iter.hasNext()) { - Entry entry = iter.next(); - final String xattrName = entry.getKey(); + for (Entry entry : srcXAttrs.entrySet()) { + String xattrName = entry.getKey(); if (xattrName.startsWith(rawNS) || preserveXAttrs) { - targetFS.setXAttr(path, entry.getKey(), entry.getValue()); + targetFS.setXAttr(path, xattrName, entry.getValue()); } } } } - if (attributes.contains(FileAttribute.REPLICATION) && ! targetFileStatus.isDirectory() && - srcFileStatus.getReplication() != targetFileStatus.getReplication()) { + if (attributes.contains(FileAttribute.REPLICATION) && !targetFileStatus.isDirectory() && + (srcFileStatus.getReplication() != targetFileStatus.getReplication())) { targetFS.setReplication(path, srcFileStatus.getReplication()); } if (attributes.contains(FileAttribute.GROUP) && - !group.equals(srcFileStatus.getGroup())) { + !group.equals(srcFileStatus.getGroup())) { group = srcFileStatus.getGroup(); chown = true; } if (attributes.contains(FileAttribute.USER) && - !user.equals(srcFileStatus.getOwner())) { + !user.equals(srcFileStatus.getOwner())) { user = srcFileStatus.getOwner(); chown = true; } @@ -256,6 +254,12 @@ public static void preserve(FileSystem targetFS, Path path, if (chown) { targetFS.setOwner(path, user, group); } + + if (attributes.contains(FileAttribute.TIMES)) { + targetFS.setTimes(path, + srcFileStatus.getModificationTime(), + srcFileStatus.getAccessTime()); + } } /** diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java index 2c42269c37e..30fb25b07c7 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java @@ -497,7 +497,7 @@ public void testPreserve() { attribIterator.next(); i++; } - Assert.assertEquals(i, 6); + Assert.assertEquals(i, DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.length() - 2); try { OptionsParser.parse(new String[] { diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java index 3f84772dd25..ec60fa800fa 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java @@ -590,6 +590,7 @@ public StubContext run() { EnumSet.allOf(DistCpOptions.FileAttribute.class); preserveStatus.remove(DistCpOptions.FileAttribute.ACL); preserveStatus.remove(DistCpOptions.FileAttribute.XATTR); + preserveStatus.remove(DistCpOptions.FileAttribute.TIMES); context.getConfiguration().set(DistCpConstants.CONF_LABEL_PRESERVE_STATUS, DistCpUtils.packAttributes(preserveStatus)); diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java index c4b64b9dffd..200d78664fb 100644 --- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java +++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/util/TestDistCpUtils.java @@ -18,38 +18,44 @@ package org.apache.hadoop.tools.util; -import org.apache.hadoop.tools.DistCpOptions.FileAttribute; +import java.io.IOException; +import java.io.OutputStream; +import java.util.EnumSet; +import java.util.Random; +import java.util.Stack; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.tools.CopyListingFileStatus; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.Assert; -import org.junit.Test; -import org.junit.BeforeClass; +import org.apache.hadoop.tools.DistCpOptionSwitch; +import org.apache.hadoop.tools.DistCpOptions.FileAttribute; import org.junit.AfterClass; - -import java.util.EnumSet; -import java.util.Random; -import java.util.Stack; -import java.io.IOException; -import java.io.OutputStream; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; public class TestDistCpUtils { private static final Log LOG = LogFactory.getLog(TestDistCpUtils.class); private static final Configuration config = new Configuration(); private static MiniDFSCluster cluster; - + private static final FsPermission fullPerm = new FsPermission((short) 777); + private static final FsPermission almostFullPerm = new FsPermission((short) 666); + private static final FsPermission noPerm = new FsPermission((short) 0); + @BeforeClass public static void create() throws IOException { - cluster = new MiniDFSCluster.Builder(config).numDataNodes(1).format(true) - .build(); + cluster = new MiniDFSCluster.Builder(config) + .numDataNodes(1) + .format(true) + .build(); } @AfterClass @@ -61,13 +67,16 @@ public static void destroy() { @Test public void testGetRelativePathRoot() { + Path root = new Path("/"); + Path child = new Path("/a"); + Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/a"); + } + + @Test + public void testGetRelativePath() { Path root = new Path("/tmp/abc"); Path child = new Path("/tmp/abc/xyz/file"); Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/xyz/file"); - - root = new Path("/"); - child = new Path("/a"); - Assert.assertEquals(DistCpUtils.getRelativePath(root, child), "/a"); } @Test @@ -77,70 +86,916 @@ public void testPackAttributes() { attributes.add(FileAttribute.REPLICATION); Assert.assertEquals(DistCpUtils.packAttributes(attributes), "R"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("R")); attributes.add(FileAttribute.BLOCKSIZE); Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RB"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RB")); attributes.add(FileAttribute.USER); - Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBU"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBU")); + attributes.add(FileAttribute.CHECKSUMTYPE); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUC"); attributes.add(FileAttribute.GROUP); - Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUG"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBUG")); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGC"); attributes.add(FileAttribute.PERMISSION); - Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGP"); - Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RBUGP")); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPC"); + + attributes.add(FileAttribute.TIMES); + Assert.assertEquals(DistCpUtils.packAttributes(attributes), "RBUGPCT"); + } + + public void testUnpackAttributes() { + EnumSet attributes = EnumSet.allOf(FileAttribute.class); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("RCBUGPAXT")); + + attributes.remove(FileAttribute.REPLICATION); + attributes.remove(FileAttribute.CHECKSUMTYPE); + attributes.remove(FileAttribute.ACL); + attributes.remove(FileAttribute.XATTR); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGPT")); + + attributes.remove(FileAttribute.TIMES); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("BUGP")); + + attributes.remove(FileAttribute.BLOCKSIZE); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UGP")); + + attributes.remove(FileAttribute.GROUP); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("UP")); + + attributes.remove(FileAttribute.USER); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("P")); + + attributes.remove(FileAttribute.PERMISSION); + Assert.assertEquals(attributes, DistCpUtils.unpackAttributes("")); } @Test - public void testPreserve() { - try { - FileSystem fs = FileSystem.get(config); - EnumSet attributes = EnumSet.noneOf(FileAttribute.class); + public void testPreserveDefaults() throws IOException { + FileSystem fs = FileSystem.get(config); + + // preserve replication, block size, user, group, permission, + // checksum type and timestamps + EnumSet attributes = + DistCpUtils.unpackAttributes( + DistCpOptionSwitch.PRESERVE_STATUS_DEFAULT.substring(1)); + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); - Path path = new Path("/tmp/abc"); - Path src = new Path("/tmp/src"); - fs.mkdirs(path); - fs.mkdirs(src); - CopyListingFileStatus srcStatus = new CopyListingFileStatus( - fs.getFileStatus(src)); + createFile(fs, src); + createFile(fs, dst); - FsPermission noPerm = new FsPermission((short) 0); - fs.setPermission(path, noPerm); - fs.setOwner(path, "nobody", "nobody"); + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); - DistCpUtils.preserve(fs, path, srcStatus, attributes, false); - FileStatus target = fs.getFileStatus(path); - Assert.assertEquals(target.getPermission(), noPerm); - Assert.assertEquals(target.getOwner(), "nobody"); - Assert.assertEquals(target.getGroup(), "nobody"); + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); - attributes.add(FileAttribute.PERMISSION); - DistCpUtils.preserve(fs, path, srcStatus, attributes, false); - target = fs.getFileStatus(path); - Assert.assertEquals(target.getPermission(), srcStatus.getPermission()); - Assert.assertEquals(target.getOwner(), "nobody"); - Assert.assertEquals(target.getGroup(), "nobody"); + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); - attributes.add(FileAttribute.GROUP); - attributes.add(FileAttribute.USER); - DistCpUtils.preserve(fs, path, srcStatus, attributes, false); - target = fs.getFileStatus(path); - Assert.assertEquals(target.getPermission(), srcStatus.getPermission()); - Assert.assertEquals(target.getOwner(), srcStatus.getOwner()); - Assert.assertEquals(target.getGroup(), srcStatus.getGroup()); + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); - fs.delete(path, true); - fs.delete(src, true); - } catch (IOException e) { - LOG.error("Exception encountered ", e); - Assert.fail("Preserve test failure"); - } + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveNothingOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.noneOf(FileAttribute.class); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(dstStatus.getAccessTime() == 100); + Assert.assertTrue(dstStatus.getModificationTime() == 100); + Assert.assertTrue(dstStatus.getReplication() == 0); + } + + @Test + public void testPreservePermissionOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.PERMISSION); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + } + + @Test + public void testPreserveGroupOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.GROUP); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); + } + + @Test + public void testPreserveUserOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.USER); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + } + + @Test + public void testPreserveReplicationOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.REPLICATION); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + // Replication shouldn't apply to dirs so this should still be 0 == 0 + Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveTimestampOnDirectory() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.TIMES); + + Path dst = new Path("/tmp/abc"); + Path src = new Path("/tmp/src"); + + createDirectory(fs, src); + createDirectory(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + } + + @Test + public void testPreserveNothingOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.noneOf(FileAttribute.class); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreservePermissionOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.PERMISSION); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertTrue(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveGroupOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.GROUP); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveUserOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.USER); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveReplicationOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.REPLICATION); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveTimestampOnFile() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.of(FileAttribute.TIMES); + + Path dst = new Path("/tmp/dest2"); + Path src = new Path("/tmp/src2"); + + createFile(fs, src); + createFile(fs, dst); + + fs.setPermission(src, fullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(dst, noPerm); + fs.setOwner(dst, "nobody", "nobody-group"); + fs.setTimes(dst, 100, 100); + fs.setReplication(dst, (short) 2); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, dst, srcStatus, attributes, false); + + CopyListingFileStatus dstStatus = new CopyListingFileStatus(fs.getFileStatus(dst)); + + // FileStatus.equals only compares path field, must explicitly compare all fields + Assert.assertFalse(srcStatus.getPermission().equals(dstStatus.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(dstStatus.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(dstStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == dstStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == dstStatus.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == dstStatus.getReplication()); + } + + @Test + public void testPreserveOnFileUpwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.allOf(FileAttribute.class); + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, f2, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> f2 ? should be yes + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertTrue(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == f2Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f0 ? should be no + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); + + // attributes of src -> d2 ? should be no + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertTrue(d2Status.getAccessTime() == 300); + Assert.assertTrue(d2Status.getModificationTime() == 300); + Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertTrue(d1Status.getAccessTime() == 400); + Assert.assertTrue(d1Status.getModificationTime() == 400); + Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication()); + } + + @Test + public void testPreserveOnDirectoryUpwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.allOf(FileAttribute.class); + + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, d2, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> d2 ? should be yes + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertTrue(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == d2Status.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == d2Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication()); + + // attributes of src -> f2 ? should be no + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f0 ? should be no + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); + } + + @Test + public void testPreserveOnFileDownwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.allOf(FileAttribute.class); + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, f0, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> f0 ? should be yes + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertTrue(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() == f0Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f2 ? should be no + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertTrue(d1Status.getAccessTime() == 400); + Assert.assertTrue(d1Status.getModificationTime() == 400); + Assert.assertFalse(srcStatus.getReplication() == d1Status.getReplication()); + + // attributes of src -> d2 ? should be no + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertTrue(d2Status.getAccessTime() == 300); + Assert.assertTrue(d2Status.getModificationTime() == 300); + Assert.assertFalse(srcStatus.getReplication() == d2Status.getReplication()); + } + + @Test + public void testPreserveOnDirectoryDownwardRecursion() throws IOException { + FileSystem fs = FileSystem.get(config); + EnumSet attributes = EnumSet.allOf(FileAttribute.class); + // Remove ACL because tests run with dfs.namenode.acls.enabled false + attributes.remove(FileAttribute.ACL); + + Path src = new Path("/tmp/src2"); + Path f0 = new Path("/f0"); + Path f1 = new Path("/d1/f1"); + Path f2 = new Path("/d1/d2/f2"); + Path d1 = new Path("/d1/"); + Path d2 = new Path("/d1/d2/"); + Path root = new Path("/"); + + createFile(fs, src); + createFile(fs, f0); + createFile(fs, f1); + createFile(fs, f2); + + fs.setPermission(src, almostFullPerm); + fs.setOwner(src, "somebody", "somebody-group"); + fs.setTimes(src, 0, 0); + fs.setReplication(src, (short) 1); + + fs.setPermission(root, fullPerm); + fs.setOwner(root, "anybody", "anybody-group"); + fs.setTimes(root, 400, 400); + fs.setReplication(root, (short) 3); + + fs.setPermission(d1, fullPerm); + fs.setOwner(d1, "anybody", "anybody-group"); + fs.setTimes(d1, 400, 400); + fs.setReplication(d1, (short) 3); + + fs.setPermission(d2, fullPerm); + fs.setOwner(d2, "anybody", "anybody-group"); + fs.setTimes(d2, 300, 300); + fs.setReplication(d2, (short) 3); + + fs.setPermission(f0, fullPerm); + fs.setOwner(f0, "anybody", "anybody-group"); + fs.setTimes(f0, 200, 200); + fs.setReplication(f0, (short) 3); + + fs.setPermission(f1, fullPerm); + fs.setOwner(f1, "anybody", "anybody-group"); + fs.setTimes(f1, 200, 200); + fs.setReplication(f1, (short) 3); + + fs.setPermission(f2, fullPerm); + fs.setOwner(f2, "anybody", "anybody-group"); + fs.setTimes(f2, 200, 200); + fs.setReplication(f2, (short) 3); + + CopyListingFileStatus srcStatus = new CopyListingFileStatus(fs.getFileStatus(src)); + + DistCpUtils.preserve(fs, root, srcStatus, attributes, false); + + cluster.triggerHeartbeats(); + + // FileStatus.equals only compares path field, must explicitly compare all fields + // attributes of src -> root ? should be yes + CopyListingFileStatus rootStatus = new CopyListingFileStatus(fs.getFileStatus(root)); + Assert.assertTrue(srcStatus.getPermission().equals(rootStatus.getPermission())); + Assert.assertTrue(srcStatus.getOwner().equals(rootStatus.getOwner())); + Assert.assertTrue(srcStatus.getGroup().equals(rootStatus.getGroup())); + Assert.assertTrue(srcStatus.getAccessTime() == rootStatus.getAccessTime()); + Assert.assertTrue(srcStatus.getModificationTime() == rootStatus.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != rootStatus.getReplication()); + + // attributes of src -> d1 ? should be no + CopyListingFileStatus d1Status = new CopyListingFileStatus(fs.getFileStatus(d1)); + Assert.assertFalse(srcStatus.getPermission().equals(d1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == d1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == d1Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d1Status.getReplication()); + + // attributes of src -> d2 ? should be no + CopyListingFileStatus d2Status = new CopyListingFileStatus(fs.getFileStatus(d2)); + Assert.assertFalse(srcStatus.getPermission().equals(d2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(d2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(d2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == d2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == d2Status.getModificationTime()); + Assert.assertTrue(srcStatus.getReplication() != d2Status.getReplication()); + + // attributes of src -> f0 ? should be no + CopyListingFileStatus f0Status = new CopyListingFileStatus(fs.getFileStatus(f0)); + Assert.assertFalse(srcStatus.getPermission().equals(f0Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f0Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f0Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f0Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f0Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f0Status.getReplication()); + + // attributes of src -> f1 ? should be no + CopyListingFileStatus f1Status = new CopyListingFileStatus(fs.getFileStatus(f1)); + Assert.assertFalse(srcStatus.getPermission().equals(f1Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f1Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f1Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f1Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f1Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f1Status.getReplication()); + + // attributes of src -> f2 ? should be no + CopyListingFileStatus f2Status = new CopyListingFileStatus(fs.getFileStatus(f2)); + Assert.assertFalse(srcStatus.getPermission().equals(f2Status.getPermission())); + Assert.assertFalse(srcStatus.getOwner().equals(f2Status.getOwner())); + Assert.assertFalse(srcStatus.getGroup().equals(f2Status.getGroup())); + Assert.assertFalse(srcStatus.getAccessTime() == f2Status.getAccessTime()); + Assert.assertFalse(srcStatus.getModificationTime() == f2Status.getModificationTime()); + Assert.assertFalse(srcStatus.getReplication() == f2Status.getReplication()); } private static Random rand = new Random(); @@ -168,10 +1023,10 @@ public static String createTestSetup(String baseDir, fs.setPermission(new Path(base + "/newTest/hello/world2/newworld"), perm); fs.setPermission(new Path(base + "/newTest/hello/world3"), perm); fs.setPermission(new Path(base + "/newTest/hello/world3/oldworld"), perm); - createFile(fs, base + "/newTest/1"); - createFile(fs, base + "/newTest/hello/2"); - createFile(fs, base + "/newTest/hello/world3/oldworld/3"); - createFile(fs, base + "/newTest/hello/world2/4"); + createFile(fs, new Path(base, "/newTest/1")); + createFile(fs, new Path(base, "/newTest/hello/2")); + createFile(fs, new Path(base, "/newTest/hello/world3/oldworld/3")); + createFile(fs, new Path(base, "/newTest/hello/world2/4")); return base; } @@ -191,12 +1046,27 @@ public static void delete(FileSystem fs, String path) { LOG.warn("Exception encountered ", e); } } - + public static void createFile(FileSystem fs, String filePath) throws IOException { - OutputStream out = fs.create(new Path(filePath)); + Path path = new Path(filePath); + createFile(fs, path); + } + + /** Creates a new, empty file at filePath and always overwrites */ + public static void createFile(FileSystem fs, Path filePath) throws IOException { + OutputStream out = fs.create(filePath, true); IOUtils.closeStream(out); } + /** Creates a new, empty directory at dirPath and always overwrites */ + public static void createDirectory(FileSystem fs, Path dirPath) throws IOException { + fs.delete(dirPath, true); + boolean created = fs.mkdirs(dirPath); + if (!created) { + LOG.warn("Could not create directory " + dirPath + " this might cause test failures."); + } + } + public static boolean checkIfFoldersAreInSync(FileSystem fs, String targetBase, String sourceBase) throws IOException { Path base = new Path(targetBase);