From 9923fa2490e0c0b38c6862299a8bcc5e5ad845ed Mon Sep 17 00:00:00 2001 From: Inigo Goiri Date: Fri, 27 Apr 2018 16:28:17 -0700 Subject: [PATCH] HDFS-13508. RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries. Contributed by Ekanth S. (cherry picked from commit 484440602c5b69fbd8106010603c61ae051056dd) --- .../hdfs/tools/federation/RouterAdmin.java | 16 +++ .../federation/router/TestRouterAdminCLI.java | 117 +++++++++++++++++- 2 files changed, 130 insertions(+), 3 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java index 17707dc334d..b0a2062a6e6 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java @@ -26,6 +26,7 @@ import java.util.Map; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; import org.apache.hadoop.hdfs.HdfsConfiguration; import org.apache.hadoop.hdfs.protocol.HdfsConstants; @@ -322,6 +323,7 @@ public class RouterAdmin extends Configured implements Tool { public boolean addMount(String mount, String[] nss, String dest, boolean readonly, DestinationOrder order, ACLEntity aclInfo) throws IOException { + mount = normalizeFileSystemPath(mount); // Get the existing entry MountTableManager mountTable = client.getMountTableManager(); GetMountTableEntriesRequest getRequest = @@ -473,6 +475,7 @@ public class RouterAdmin extends Configured implements Tool { public boolean updateMount(String mount, String[] nss, String dest, boolean readonly, DestinationOrder order, ACLEntity aclInfo) throws IOException { + mount = normalizeFileSystemPath(mount); MountTableManager mountTable = client.getMountTableManager(); // Create a new entry @@ -519,6 +522,7 @@ public class RouterAdmin extends Configured implements Tool { * @throws IOException If it cannot be removed. */ public boolean removeMount(String path) throws IOException { + path = normalizeFileSystemPath(path); MountTableManager mountTable = client.getMountTableManager(); RemoveMountTableEntryRequest request = RemoveMountTableEntryRequest.newInstance(path); @@ -538,6 +542,7 @@ public class RouterAdmin extends Configured implements Tool { * @throws IOException If it cannot be listed. */ public void listMounts(String path) throws IOException { + path = normalizeFileSystemPath(path); MountTableManager mountTable = client.getMountTableManager(); GetMountTableEntriesRequest request = GetMountTableEntriesRequest.newInstance(path); @@ -797,6 +802,17 @@ public class RouterAdmin extends Configured implements Tool { } } + /** + * Normalize a path for that filesystem. + * + * @param path Path to normalize. + * @return Normalized path. + */ + private static String normalizeFileSystemPath(final String path) { + Path normalizedPath = new Path(path); + return normalizedPath.toString(); + } + /** * Inner class that stores ACL info of mount table. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java index 4e84c33b2d2..2537c193c04 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java +++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java @@ -158,6 +158,45 @@ public class TestRouterAdminCLI { assertTrue(mountTable.isReadOnly()); } + @Test + public void testAddMountTableNotNormalized() throws Exception { + String nsId = "ns0"; + String src = "/test-addmounttable-notnormalized"; + String srcWithSlash = src + "/"; + String dest = "/addmounttable-notnormalized"; + String[] argv = new String[] {"-add", srcWithSlash, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest + .newInstance(src); + GetMountTableEntriesResponse getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + MountTable mountTable = getResponse.getEntries().get(0); + + List destinations = mountTable.getDestinations(); + assertEquals(1, destinations.size()); + + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, destinations.get(0).getNameserviceId()); + assertEquals(dest, destinations.get(0).getDest()); + assertFalse(mountTable.isReadOnly()); + + // test mount table update behavior + dest = dest + "-new"; + argv = new String[] {"-add", srcWithSlash, nsId, dest, "-readonly"}; + assertEquals(0, ToolRunner.run(admin, argv)); + stateStore.loadCache(MountTableStoreImpl.class, true); + + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + mountTable = getResponse.getEntries().get(0); + assertEquals(2, mountTable.getDestinations().size()); + assertEquals(nsId, mountTable.getDestinations().get(1).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(1).getDest()); + assertTrue(mountTable.isReadOnly()); + } + @Test public void testAddOrderMountTable() throws Exception { testAddOrderMountTable(DestinationOrder.HASH); @@ -192,6 +231,7 @@ public class TestRouterAdminCLI { public void testListMountTable() throws Exception { String nsId = "ns0"; String src = "/test-lsmounttable"; + String srcWithSlash = src + "/"; String dest = "/lsmounttable"; String[] argv = new String[] {"-add", src, nsId, dest}; assertEquals(0, ToolRunner.run(admin, argv)); @@ -203,6 +243,11 @@ public class TestRouterAdminCLI { assertEquals(0, ToolRunner.run(admin, argv)); assertTrue(out.toString().contains(src)); + // Test with not-normalized src input + argv = new String[] {"-ls", srcWithSlash}; + assertEquals(0, ToolRunner.run(admin, argv)); + assertTrue(out.toString().contains(src)); + out.reset(); GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest .newInstance("/"); @@ -255,6 +300,33 @@ public class TestRouterAdminCLI { "Cannot remove mount point " + invalidPath)); } + @Test + public void testRemoveMountTableNotNormalized() throws Exception { + String nsId = "ns0"; + String src = "/test-rmmounttable-notnormalized"; + String srcWithSlash = src + "/"; + String dest = "/rmmounttable-notnormalized"; + String[] argv = new String[] {"-add", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest + .newInstance(src); + GetMountTableEntriesResponse getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + // ensure mount table added successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + + argv = new String[] {"-rm", srcWithSlash}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + assertEquals(0, getResponse.getEntries().size()); + } + @Test public void testMountTableDefaultACL() throws Exception { String[] argv = new String[] {"-add", "/testpath0", "ns0", "/testdir0"}; @@ -552,12 +624,12 @@ public class TestRouterAdminCLI { } @Test - public void testUpdateNameserviceDestinationForExistingMountTable() throws + public void testUpdateDestinationForExistingMountTable() throws Exception { // Add a mount table firstly String nsId = "ns0"; - String src = "/test-updateNameserviceDestinationForExistingMountTable"; - String dest = "/UpdateNameserviceDestinationForExistingMountTable"; + String src = "/test-updateDestinationForExistingMountTable"; + String dest = "/UpdateDestinationForExistingMountTable"; String[] argv = new String[] {"-add", src, nsId, dest}; assertEquals(0, ToolRunner.run(admin, argv)); @@ -589,6 +661,45 @@ public class TestRouterAdminCLI { assertEquals(newDest, mountTable.getDestinations().get(0).getDest()); } + @Test + public void testUpdateDestinationForExistingMountTableNotNormalized() throws + Exception { + // Add a mount table firstly + String nsId = "ns0"; + String src = "/test-updateDestinationForExistingMountTableNotNormalized"; + String srcWithSlash = src + "/"; + String dest = "/UpdateDestinationForExistingMountTableNotNormalized"; + String[] argv = new String[] {"-add", src, nsId, dest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + GetMountTableEntriesRequest getRequest = + GetMountTableEntriesRequest.newInstance(src); + GetMountTableEntriesResponse getResponse = + client.getMountTableManager().getMountTableEntries(getRequest); + // Ensure mount table added successfully + MountTable mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(dest, mountTable.getDestinations().get(0).getDest()); + + // Update the destination + String newNsId = "ns1"; + String newDest = "/newDestination"; + argv = new String[] {"-update", srcWithSlash, newNsId, newDest}; + assertEquals(0, ToolRunner.run(admin, argv)); + + stateStore.loadCache(MountTableStoreImpl.class, true); + getResponse = client.getMountTableManager() + .getMountTableEntries(getRequest); + // Ensure the destination updated successfully + mountTable = getResponse.getEntries().get(0); + assertEquals(src, mountTable.getSourcePath()); + assertEquals(newNsId, + mountTable.getDestinations().get(0).getNameserviceId()); + assertEquals(newDest, mountTable.getDestinations().get(0).getDest()); + } + @Test public void testUpdateReadonlyUserGroupPermissionMountable() throws Exception {