Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13508. RBF: Normalize paths (automatically) when adding, updating, removing or listing mount table entries. Contributed by Ekanth S.

This commit is contained in:
Inigo Goiri 2018-04-27 16:28:17 -07:00
parent f469628bba
commit 484440602c
2 changed files with 130 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
View File

@ -26,6 +26,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@ -322,6 +323,7 @@ public class RouterAdmin extends Configured implements Tool {
public boolean addMount(String mount, String[] nss, String dest,
boolean readonly, DestinationOrder order, ACLEntity aclInfo)
throws IOException {
mount = normalizeFileSystemPath(mount);
// Get the existing entry
MountTableManager mountTable = client.getMountTableManager();
GetMountTableEntriesRequest getRequest =
@ -473,6 +475,7 @@ public class RouterAdmin extends Configured implements Tool {
public boolean updateMount(String mount, String[] nss, String dest,
boolean readonly, DestinationOrder order, ACLEntity aclInfo)
throws IOException {
mount = normalizeFileSystemPath(mount);
MountTableManager mountTable = client.getMountTableManager();
// Create a new entry
@ -519,6 +522,7 @@ public class RouterAdmin extends Configured implements Tool {
* @throws IOException If it cannot be removed.
*/
public boolean removeMount(String path) throws IOException {
path = normalizeFileSystemPath(path);
MountTableManager mountTable = client.getMountTableManager();
RemoveMountTableEntryRequest request =
RemoveMountTableEntryRequest.newInstance(path);
@ -538,6 +542,7 @@ public class RouterAdmin extends Configured implements Tool {
* @throws IOException If it cannot be listed.
*/
public void listMounts(String path) throws IOException {
path = normalizeFileSystemPath(path);
MountTableManager mountTable = client.getMountTableManager();
GetMountTableEntriesRequest request =
GetMountTableEntriesRequest.newInstance(path);
@ -797,6 +802,17 @@ public class RouterAdmin extends Configured implements Tool {
}
}
/**
* Normalize a path for that filesystem.
*
* @param path Path to normalize.
* @return Normalized path.
*/
private static String normalizeFileSystemPath(final String path) {
Path normalizedPath = new Path(path);
return normalizedPath.toString();
}
/**
* Inner class that stores ACL info of mount table.
*/

117
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
View File

@ -158,6 +158,45 @@ public class TestRouterAdminCLI {
assertTrue(mountTable.isReadOnly());
}
@Test
public void testAddMountTableNotNormalized() throws Exception {
String nsId = "ns0";
String src = "/test-addmounttable-notnormalized";
String srcWithSlash = src + "/";
String dest = "/addmounttable-notnormalized";
String[] argv = new String[] {"-add", srcWithSlash, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
.newInstance(src);
GetMountTableEntriesResponse getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
MountTable mountTable = getResponse.getEntries().get(0);
List<RemoteLocation> destinations = mountTable.getDestinations();
assertEquals(1, destinations.size());
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, destinations.get(0).getNameserviceId());
assertEquals(dest, destinations.get(0).getDest());
assertFalse(mountTable.isReadOnly());
// test mount table update behavior
dest = dest + "-new";
argv = new String[] {"-add", srcWithSlash, nsId, dest, "-readonly"};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
mountTable = getResponse.getEntries().get(0);
assertEquals(2, mountTable.getDestinations().size());
assertEquals(nsId, mountTable.getDestinations().get(1).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(1).getDest());
assertTrue(mountTable.isReadOnly());
}
@Test
public void testAddOrderMountTable() throws Exception {
testAddOrderMountTable(DestinationOrder.HASH);
@ -192,6 +231,7 @@ public class TestRouterAdminCLI {
public void testListMountTable() throws Exception {
String nsId = "ns0";
String src = "/test-lsmounttable";
String srcWithSlash = src + "/";
String dest = "/lsmounttable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
@ -203,6 +243,11 @@ public class TestRouterAdminCLI {
assertEquals(0, ToolRunner.run(admin, argv));
assertTrue(out.toString().contains(src));
// Test with not-normalized src input
argv = new String[] {"-ls", srcWithSlash};
assertEquals(0, ToolRunner.run(admin, argv));
assertTrue(out.toString().contains(src));
out.reset();
GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
.newInstance("/");
@ -255,6 +300,33 @@ public class TestRouterAdminCLI {
"Cannot remove mount point " + invalidPath));
}
@Test
public void testRemoveMountTableNotNormalized() throws Exception {
String nsId = "ns0";
String src = "/test-rmmounttable-notnormalized";
String srcWithSlash = src + "/";
String dest = "/rmmounttable-notnormalized";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
GetMountTableEntriesRequest getRequest = GetMountTableEntriesRequest
.newInstance(src);
GetMountTableEntriesResponse getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
// ensure mount table added successfully
MountTable mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
argv = new String[] {"-rm", srcWithSlash};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
assertEquals(0, getResponse.getEntries().size());
}
@Test
public void testMountTableDefaultACL() throws Exception {
String[] argv = new String[] {"-add", "/testpath0", "ns0", "/testdir0"};
@ -552,12 +624,12 @@ public class TestRouterAdminCLI {
}
@Test
public void testUpdateNameserviceDestinationForExistingMountTable() throws
public void testUpdateDestinationForExistingMountTable() throws
Exception {
// Add a mount table firstly
String nsId = "ns0";
String src = "/test-updateNameserviceDestinationForExistingMountTable";
String dest = "/UpdateNameserviceDestinationForExistingMountTable";
String src = "/test-updateDestinationForExistingMountTable";
String dest = "/UpdateDestinationForExistingMountTable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
@ -589,6 +661,45 @@ public class TestRouterAdminCLI {
assertEquals(newDest, mountTable.getDestinations().get(0).getDest());
}
@Test
public void testUpdateDestinationForExistingMountTableNotNormalized() throws
Exception {
// Add a mount table firstly
String nsId = "ns0";
String src = "/test-updateDestinationForExistingMountTableNotNormalized";
String srcWithSlash = src + "/";
String dest = "/UpdateDestinationForExistingMountTableNotNormalized";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
GetMountTableEntriesRequest getRequest =
GetMountTableEntriesRequest.newInstance(src);
GetMountTableEntriesResponse getResponse =
client.getMountTableManager().getMountTableEntries(getRequest);
// Ensure mount table added successfully
MountTable mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
// Update the destination
String newNsId = "ns1";
String newDest = "/newDestination";
argv = new String[] {"-update", srcWithSlash, newNsId, newDest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
// Ensure the destination updated successfully
mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(newNsId,
mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(newDest, mountTable.getDestinations().get(0).getDest());
}
@Test
public void testUpdateReadonlyUserGroupPermissionMountable()
throws Exception {