From 86325e43461be112b58604ff22fc4eef0b2936ab Mon Sep 17 00:00:00 2001 From: Andrew Wang Date: Fri, 14 Oct 2016 11:41:29 -0700 Subject: [PATCH] HDFS-10883. 's behavior is not consistent in DFS after enabling EZ. Contributed by Yuanbo Liu. (cherry picked from commit 0007360c3344b3485fa17de0fd2015a628de947c) --- .../hadoop/hdfs/DistributedFileSystem.java | 5 +- .../site/markdown/TransparentEncryption.md | 4 +- .../hadoop/hdfs/TestEncryptionZones.java | 10 +- .../namenode/TestNestedEncryptionZones.java | 175 +++++++++++++----- 4 files changed, 139 insertions(+), 55 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 8b35a5f56f7..8bda7fa9dd6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -2474,11 +2474,12 @@ public DFSInotifyEventInputStream getInotifyEventStream(long lastReadTxid) */ @Override public Path getTrashRoot(Path path) { - if ((path == null) || path.isRoot() || !dfs.isHDFSEncryptionEnabled()) { + if ((path == null) || !dfs.isHDFSEncryptionEnabled()) { return super.getTrashRoot(path); } - String parentSrc = path.getParent().toUri().getPath(); + String parentSrc = path.isRoot()? + path.toUri().getPath():path.getParent().toUri().getPath(); try { EncryptionZone ez = dfs.getEZForPath(parentSrc); if ((ez != null)) { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md index e7d9f1d20d6..b82b400a3e3 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md +++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md @@ -242,12 +242,14 @@ By default, distcp compares checksums provided by the filesystem to verify that Rename and Trash considerations --------------------- -HDFS restricts file and directory renames across encryption zone boundaries. This includes renaming an encrypted file / directory into an unencrypted directory (e.g., `hdfs dfs mv /zone/encryptedFile /home/bob`), renaming an unencrypted file / directory into an encryption zone (e.g., `hdfs dfs mv /home/bob/unEncryptedFile /zone`), and renaming between two different encryption zones (e.g., `hdfs dfs mv /home/alice/zone1/foo /home/alice/zone2`). In these examples, `/zone`, `/home/alice/zone1`, and `/home/alice/zone2` are encryption zones, while `/home/bob` is not. A rename is only allowed if the source and destination paths are in the same encryption zone, or both paths are unencrypted (not in any encryption zone). +HDFS restricts file and directory renames across encryption zone boundaries. This includes renaming an encrypted file / directory into an unencrypted directory (e.g., `hdfs dfs mv /zone/encryptedFile /home/bob`), renaming an unencrypted file or directory into an encryption zone (e.g., `hdfs dfs mv /home/bob/unEncryptedFile /zone`), and renaming between two different encryption zones (e.g., `hdfs dfs mv /home/alice/zone1/foo /home/alice/zone2`). In these examples, `/zone`, `/home/alice/zone1`, and `/home/alice/zone2` are encryption zones, while `/home/bob` is not. A rename is only allowed if the source and destination paths are in the same encryption zone, or both paths are unencrypted (not in any encryption zone). This restriction enhances security and eases system management significantly. All file EDEKs under an encryption zone are encrypted with the encryption zone key. Therefore, if the encryption zone key is compromised, it is important to identify all vulnerable files and re-encrypt them. This is fundamentally difficult if a file initially created in an encryption zone can be renamed to an arbitrary location in the filesystem. To comply with the above rule, each encryption zone has its own `.Trash` directory under the "zone directory". E.g., after `hdfs dfs rm /zone/encryptedFile`, `encryptedFile` will be moved to `/zone/.Trash`, instead of the `.Trash` directory under the user's home directory. When the entire encryption zone is deleted, the "zone directory" will be moved to the `.Trash` directory under the user's home directory. +If the encryption zone is the root directory (e.g., `/` directory), the trash path of root directory is `/.Trash`, not the `.Trash` directory under the user's home directory, and the behavior of renaming sub-directories or sub-files in root directory will keep consistent with the behavior in a general encryption zone, such as `/zone` which is mentioned at the top of this section. + The `crypto` command before Hadoop 2.8.0 does not provision the `.Trash` directory automatically. If an encryption zone is created before Hadoop 2.8.0, and then the cluster is upgraded to Hadoop 2.8.0 or above, the trash directory can be provisioned using `-provisionTrash` option (e.g., `hdfs crypto -provisionTrash -path /zone`). Attack vectors -------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java index f4dc2fa8f16..f1925e132cb 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java @@ -1566,7 +1566,8 @@ public void testEncryptionZoneWithTrash() throws Exception { public void testRootDirEZTrash() throws Exception { final HdfsAdmin dfsAdmin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf); - dfsAdmin.createEncryptionZone(new Path("/"), TEST_KEY, NO_TRASH); + final Path rootDir = new Path("/"); + dfsAdmin.createEncryptionZone(rootDir, TEST_KEY, NO_TRASH); final Path encFile = new Path("/encFile"); final int len = 8192; DFSTestUtil.createFile(fs, encFile, len, (short) 1, 0xFEED); @@ -1574,6 +1575,13 @@ public void testRootDirEZTrash() throws Exception { clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1); FsShell shell = new FsShell(clientConf); verifyShellDeleteWithTrash(shell, encFile); + + // Trash path should be consistent + // if root path is an encryption zone + Path encFileTrash = shell.getCurrentTrashDir(encFile); + Path rootDirTrash = shell.getCurrentTrashDir(rootDir); + assertEquals("Root trash should be equal with ezFile trash", + encFileTrash, rootDirTrash); } @Test(timeout = 120000) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java index 13fc985180d..59d980c0418 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java @@ -20,7 +20,9 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.crypto.key.JavaKeyStoreProvider; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileSystemTestHelper; +import org.apache.hadoop.fs.FsShell; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.DFSConfigKeys; @@ -29,6 +31,8 @@ import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.protocol.HdfsConstants; +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.ToolRunner; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.junit.Before; @@ -53,16 +57,19 @@ public class TestNestedEncryptionZones { private final Path rootDir = new Path("/"); private final Path rawDir = new Path("/.reserved/raw/"); - private final Path topEZDir = new Path(rootDir, "topEZ"); - private final Path nestedEZDir = new Path(topEZDir, "nestedEZ"); - private final Path topEZBaseFile = new Path(rootDir, "topEZBaseFile"); - private Path topEZFile = new Path(topEZDir, "file"); - private Path topEZRawFile = new Path(rawDir, "topEZ/file"); + private Path nestedEZBaseFile = new Path(rootDir, "nestedEZBaseFile"); + private Path topEZBaseFile = new Path(rootDir, "topEZBaseFile"); + + private Path topEZDir; + private Path nestedEZDir; + + private Path topEZFile; + private Path nestedEZFile; + + private Path topEZRawFile; + private Path nestedEZRawFile; - private final Path nestedEZBaseFile = new Path(rootDir, "nestedEZBaseFile"); - private Path nestedEZFile = new Path(nestedEZDir, "file"); - private Path nestedEZRawFile = new Path(rawDir, "topEZ/nestedEZ/file"); // File length private final int len = 8196; @@ -92,6 +99,8 @@ public void setup() throws Exception { // Lower the batch size for testing conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES, 2); + // enable trash for testing + conf.setLong(DFSConfigKeys.FS_TRASH_INTERVAL_KEY, 1); cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build(); Logger.getLogger(EncryptionZoneManager.class).setLevel(Level.TRACE); fs = cluster.getFileSystem(); @@ -100,24 +109,17 @@ public void setup() throws Exception { // Create test keys and EZs DFSTestUtil.createKey(TOP_EZ_KEY, cluster, conf); DFSTestUtil.createKey(NESTED_EZ_KEY, cluster, conf); - fs.mkdir(topEZDir, FsPermission.getDirDefault()); - fs.createEncryptionZone(topEZDir, TOP_EZ_KEY); - fs.mkdir(nestedEZDir, FsPermission.getDirDefault()); - fs.createEncryptionZone(nestedEZDir, NESTED_EZ_KEY); - - DFSTestUtil.createFile(fs, topEZBaseFile, len, (short) 1, 0xFEED); - DFSTestUtil.createFile(fs, topEZFile, len, (short) 1, 0xFEED); - DFSTestUtil.createFile(fs, nestedEZBaseFile, len, (short) 1, 0xFEED); - DFSTestUtil.createFile(fs, nestedEZFile, len, (short) 1, 0xFEED); } @Test(timeout = 60000) public void testNestedEncryptionZones() throws Exception { + initTopEZDirAndNestedEZDir(new Path(rootDir, "topEZ")); verifyEncryption(); // Restart NameNode to test if nested EZs can be loaded from edit logs cluster.restartNameNodes(); cluster.waitActive(); + fs = cluster.getFileSystem(); verifyEncryption(); // Checkpoint and restart NameNode, to test if nested EZs can be loaded @@ -127,21 +129,88 @@ public void testNestedEncryptionZones() throws Exception { fs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE); cluster.restartNameNodes(); cluster.waitActive(); + fs = cluster.getFileSystem(); verifyEncryption(); + renameChildrenOfEZ(); + + // Verify that a non-nested EZ cannot be moved into another EZ + Path topEZ2Dir = new Path(rootDir, "topEZ2"); + fs.mkdir(topEZ2Dir, FsPermission.getDirDefault()); + fs.createEncryptionZone(topEZ2Dir, TOP_EZ_KEY); + try { + fs.rename(topEZ2Dir, new Path(topEZDir, "topEZ2")); + fail("Shouldn't be able to move a non-nested EZ into another " + + "existing EZ."); + } catch (Exception e){ + assertTrue(e.getMessage().contains( + "can't be moved into an encryption zone")); + } + + // Should be able to rename the root dir of an EZ. + fs.rename(topEZDir, new Path(rootDir, "newTopEZ")); + + // Should be able to rename the nested EZ dir within the same top EZ. + fs.rename(new Path(rootDir, "newTopEZ/nestedEZ"), + new Path(rootDir, "newTopEZ/newNestedEZ")); + } + + @Test(timeout = 60000) + public void testNestedEZWithRoot() throws Exception { + initTopEZDirAndNestedEZDir(rootDir); + verifyEncryption(); + + // test rename file + renameChildrenOfEZ(); + + final String currentUser = + UserGroupInformation.getCurrentUser().getShortUserName(); + final Path suffixTrashPath = new Path( + FileSystem.TRASH_PREFIX, currentUser); + + final Path rootTrash = fs.getTrashRoot(rootDir); + final Path topEZTrash = fs.getTrashRoot(topEZFile); + final Path nestedEZTrash = fs.getTrashRoot(nestedEZFile); + + final Path expectedTopEZTrash = fs.makeQualified( + new Path(topEZDir, suffixTrashPath)); + final Path expectedNestedEZTrash = fs.makeQualified( + new Path(nestedEZDir, suffixTrashPath)); + + assertEquals("Top ez trash should be " + expectedTopEZTrash, + expectedTopEZTrash, topEZTrash); + assertEquals("Root trash should be equal with TopEZFile trash", + topEZTrash, rootTrash); + assertEquals("Nested ez Trash should be " + expectedNestedEZTrash, + expectedNestedEZTrash, nestedEZTrash); + + // delete rename file and test trash + FsShell shell = new FsShell(fs.getConf()); + final Path topTrashFile = new Path( + shell.getCurrentTrashDir(topEZFile) + "/" + topEZFile); + final Path nestedTrashFile = new Path( + shell.getCurrentTrashDir(nestedEZFile) + "/" + nestedEZFile); + + ToolRunner.run(shell, new String[]{"-rm", topEZFile.toString()}); + ToolRunner.run(shell, new String[]{"-rm", nestedEZFile.toString()}); + + assertTrue("File not in trash : " + topTrashFile, fs.exists(topTrashFile)); + assertTrue( + "File not in trash : " + nestedTrashFile, fs.exists(nestedTrashFile)); + } + + private void renameChildrenOfEZ() throws Exception{ Path renamedTopEZFile = new Path(topEZDir, "renamedFile"); Path renamedNestedEZFile = new Path(nestedEZDir, "renamedFile"); - try { - fs.rename(topEZFile, renamedTopEZFile); - fs.rename(nestedEZFile, renamedNestedEZFile); - } catch (Exception e) { - fail("Should be able to rename files within the same EZ."); - } + + //Should be able to rename files within the same EZ. + fs.rename(topEZFile, renamedTopEZFile); + fs.rename(nestedEZFile, renamedNestedEZFile); topEZFile = renamedTopEZFile; nestedEZFile = renamedNestedEZFile; - topEZRawFile = new Path(rawDir, "topEZ/renamedFile"); - nestedEZRawFile = new Path(rawDir, "topEZ/nestedEZ/renamedFile"); + topEZRawFile = new Path(rawDir + topEZFile.toUri().getPath()); + nestedEZRawFile = new Path(rawDir + nestedEZFile.toUri().getPath()); verifyEncryption(); // Verify that files in top EZ cannot be moved into the nested EZ, and @@ -168,36 +237,40 @@ public void testNestedEncryptionZones() throws Exception { fs.rename(nestedEZFile, new Path(rootDir, "movedNestedEZFile")); fail("Shouldn't be able to move the nested EZ out of the top EZ."); } catch (Exception e) { - assertTrue(e.getMessage().contains( - "can't be moved from an encryption zone")); + String exceptionMsg = e.getMessage(); + assertTrue(exceptionMsg.contains( + "can't be moved from") && exceptionMsg.contains("encryption zone")); } + } - // Verify that a non-nested EZ cannot be moved into another EZ - Path topEZ2Dir = new Path(rootDir, "topEZ2"); - fs.mkdir(topEZ2Dir, FsPermission.getDirDefault()); - fs.createEncryptionZone(topEZ2Dir, TOP_EZ_KEY); - try { - fs.rename(topEZ2Dir, new Path(topEZDir, "topEZ2")); - fail("Shouldn't be able to move a non-nested EZ into another " + - "existing EZ."); - } catch (Exception e){ - assertTrue(e.getMessage().contains( - "can't be moved into an encryption zone")); - } + private void initTopEZDirAndNestedEZDir(Path topPath) throws Exception { - try { - fs.rename(topEZDir, new Path(rootDir, "newTopEZDir")); - } catch (Exception e) { - fail("Should be able to rename the root dir of an EZ."); - } + // init fs root directory + fs.delete(rootDir, true); + + + // init top and nested path or file + topEZDir = topPath; + nestedEZDir = new Path(topEZDir, "nestedEZ"); + + topEZFile = new Path(topEZDir, "file"); + nestedEZFile = new Path(nestedEZDir, "file"); + + topEZRawFile = new Path(rawDir + topEZFile.toUri().getPath()); + nestedEZRawFile = new Path(rawDir + nestedEZFile.toUri().getPath()); + + // create ez zone + fs.mkdir(topEZDir, FsPermission.getDirDefault()); + fs.createEncryptionZone(topEZDir, TOP_EZ_KEY); + fs.mkdir(nestedEZDir, FsPermission.getDirDefault()); + fs.createEncryptionZone(nestedEZDir, NESTED_EZ_KEY); + + // create files + DFSTestUtil.createFile(fs, topEZBaseFile, len, (short) 1, 0xFEED); + DFSTestUtil.createFile(fs, topEZFile, len, (short) 1, 0xFEED); + DFSTestUtil.createFile(fs, nestedEZBaseFile, len, (short) 1, 0xFEED); + DFSTestUtil.createFile(fs, nestedEZFile, len, (short) 1, 0xFEED); - try { - fs.rename(new Path(rootDir, "newTopEZDir/nestedEZDir"), - new Path(rootDir, "newTopEZDir/newNestedEZDir")); - } catch (Exception e) { - fail("Should be able to rename the nested EZ dir within " + - "the same top EZ."); - } } private void verifyEncryption() throws Exception {