Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-13326. RBF: Improve the interfaces to modify and view mount tables. Contributed by Gang Li.

This commit is contained in:
Inigo Goiri 2018-04-26 12:59:22 -07:00 committed by Owen O'Malley
parent 598483e496
commit 09c9cf1d43
3 changed files with 276 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/tools/federation/RouterAdmin.java
View File

@ -94,7 +94,10 @@ public class RouterAdmin extends Configured implements Tool {
*/
public void printUsage() {
String usage = "Federation Admin Tools:\n"
+ "\t[-add <source> <nameservice> <destination> "
+ "\t[-add <source> <nameservice1, nameservice2, ...> <destination> "
+ "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] "
+ "-owner <owner> -group <group> -mode <mode>]\n"
+ "\t[-update <source> <nameservice1, nameservice2, ...> <destination> "
+ "[-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] "
+ "-owner <owner> -group <group> -mode <mode>]\n"
+ "\t[-rm <source>]\n"
@ -112,7 +115,7 @@ public class RouterAdmin extends Configured implements Tool {
@Override
public int run(String[] argv) throws Exception {
if (argv.length < 1) {
System.err.println("Not enough parameters specificed");
System.err.println("Not enough parameters specified");
printUsage();
return -1;
}
@ -124,31 +127,37 @@ public class RouterAdmin extends Configured implements Tool {
// Verify that we have enough command line parameters
if ("-add".equals(cmd)) {
if (argv.length < 4) {
System.err.println("Not enough parameters specificed for cmd " + cmd);
System.err.println("Not enough parameters specified for cmd " + cmd);
printUsage();
return exitCode;
}
} else if ("-update".equals(cmd)) {
if (argv.length < 4) {
System.err.println("Not enough parameters specified for cmd " + cmd);
printUsage();
return exitCode;
}
} else if ("-rm".equalsIgnoreCase(cmd)) {
if (argv.length < 2) {
System.err.println("Not enough parameters specificed for cmd " + cmd);
System.err.println("Not enough parameters specified for cmd " + cmd);
printUsage();
return exitCode;
}
} else if ("-setQuota".equalsIgnoreCase(cmd)) {
if (argv.length < 4) {
System.err.println("Not enough parameters specificed for cmd " + cmd);
System.err.println("Not enough parameters specified for cmd " + cmd);
printUsage();
return exitCode;
}
} else if ("-clrQuota".equalsIgnoreCase(cmd)) {
if (argv.length < 2) {
System.err.println("Not enough parameters specificed for cmd " + cmd);
System.err.println("Not enough parameters specified for cmd " + cmd);
printUsage();
return exitCode;
}
} else if ("-safemode".equalsIgnoreCase(cmd)) {
if (argv.length < 2) {
System.err.println("Not enough parameters specificed for cmd " + cmd);
System.err.println("Not enough parameters specified for cmd " + cmd);
printUsage();
return exitCode;
}
@ -181,7 +190,11 @@ public class RouterAdmin extends Configured implements Tool {
try {
if ("-add".equals(cmd)) {
if (addMount(argv, i)) {
System.out.println("Successfuly added mount point " + argv[i]);
System.out.println("Successfully added mount point " + argv[i]);
}
} else if ("-update".equals(cmd)) {
if (updateMount(argv, i)) {
System.out.println("Successfully updated mount point " + argv[i]);
}
} else if ("-rm".equals(cmd)) {
if (removeMount(argv[i])) {
@ -398,6 +411,107 @@ public class RouterAdmin extends Configured implements Tool {
}
}
/**
* Update a mount table entry.
*
* @param parameters Parameters for the mount point.
* @param i Index in the parameters.
*/
public boolean updateMount(String[] parameters, int i) throws IOException {
// Mandatory parameters
String mount = parameters[i++];
String[] nss = parameters[i++].split(",");
String dest = parameters[i++];
// Optional parameters
boolean readOnly = false;
String owner = null;
String group = null;
FsPermission mode = null;
DestinationOrder order = null;
while (i < parameters.length) {
if (parameters[i].equals("-readonly")) {
readOnly = true;
} else if (parameters[i].equals("-order")) {
i++;
try {
order = DestinationOrder.valueOf(parameters[i]);
} catch(Exception e) {
System.err.println("Cannot parse order: " + parameters[i]);
}
} else if (parameters[i].equals("-owner")) {
i++;
owner = parameters[i];
} else if (parameters[i].equals("-group")) {
i++;
group = parameters[i];
} else if (parameters[i].equals("-mode")) {
i++;
short modeValue = Short.parseShort(parameters[i], 8);
mode = new FsPermission(modeValue);
}
i++;
}
return updateMount(mount, nss, dest, readOnly, order,
new ACLEntity(owner, group, mode));
}
/**
* Update a mount table entry.
*
* @param mount Mount point.
* @param nss Nameservices where this is mounted to.
* @param dest Destination path.
* @param readonly If the mount point is read only.
* @param order Order of the destination locations.
* @param aclInfo the ACL info for mount point.
* @return If the mount point was updated.
* @throws IOException Error updating the mount point.
*/
public boolean updateMount(String mount, String[] nss, String dest,
boolean readonly, DestinationOrder order, ACLEntity aclInfo)
throws IOException {
MountTableManager mountTable = client.getMountTableManager();
// Create a new entry
Map<String, String> destMap = new LinkedHashMap<>();
for (String ns : nss) {
destMap.put(ns, dest);
}
MountTable newEntry = MountTable.newInstance(mount, destMap);
newEntry.setReadOnly(readonly);
if (order != null) {
newEntry.setDestOrder(order);
}
// Update ACL info of mount table entry
if (aclInfo.getOwner() != null) {
newEntry.setOwnerName(aclInfo.getOwner());
}
if (aclInfo.getGroup() != null) {
newEntry.setGroupName(aclInfo.getGroup());
}
if (aclInfo.getMode() != null) {
newEntry.setMode(aclInfo.getMode());
}
UpdateMountTableEntryRequest updateRequest =
UpdateMountTableEntryRequest.newInstance(newEntry);
UpdateMountTableEntryResponse updateResponse =
mountTable.updateMountTableEntry(updateRequest);
boolean updated = updateResponse.getStatus();
if (!updated) {
System.err.println("Cannot update mount point " + mount);
}
return updated;
}
/**
* Remove mount point.
*

150
hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterAdminCLI.java
View File

@ -529,4 +529,154 @@ public class TestRouterAdminCLI {
}
}, 1000, 30000);
}
@Test
public void testUpdateNonExistingMountTable() throws Exception {
System.setOut(new PrintStream(out));
String nsId = "ns0";
String src = "/test-updateNonExistingMounttable";
String dest = "/updateNonExistingMounttable";
String[] argv = new String[] {"-update", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
GetMountTableEntriesRequest getRequest =
GetMountTableEntriesRequest.newInstance(src);
GetMountTableEntriesResponse getResponse =
client.getMountTableManager().getMountTableEntries(getRequest);
// Ensure the destination updated successfully
MountTable mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
}
@Test
public void testUpdateNameserviceDestinationForExistingMountTable() throws
Exception {
// Add a mount table firstly
String nsId = "ns0";
String src = "/test-updateNameserviceDestinationForExistingMountTable";
String dest = "/UpdateNameserviceDestinationForExistingMountTable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
GetMountTableEntriesRequest getRequest =
GetMountTableEntriesRequest.newInstance(src);
GetMountTableEntriesResponse getResponse =
client.getMountTableManager().getMountTableEntries(getRequest);
// Ensure mount table added successfully
MountTable mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
// Update the destination
String newNsId = "ns1";
String newDest = "/newDestination";
argv = new String[] {"-update", src, newNsId, newDest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
// Ensure the destination updated successfully
mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(newNsId,
mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(newDest, mountTable.getDestinations().get(0).getDest());
}
@Test
public void testUpdateReadonlyUserGroupPermissionMountable()
throws Exception {
// Add a mount table
String nsId = "ns0";
String src = "/test-updateReadonlyUserGroupPermissionMountTable";
String dest = "/UpdateReadonlyUserGroupPermissionMountTable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
GetMountTableEntriesRequest getRequest =
GetMountTableEntriesRequest.newInstance(src);
GetMountTableEntriesResponse getResponse =
client.getMountTableManager().getMountTableEntries(getRequest);
// Ensure mount table added successfully
MountTable mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
assertFalse(mountTable.isReadOnly());
// Update the readonly, owner, group and permission
String testOwner = "test_owner";
String testGroup = "test_group";
argv = new String[] {"-update", src, nsId, dest, "-readonly",
"-owner", testOwner, "-group", testGroup, "-mode", "0455"};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
// Ensure the destination updated successfully
mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
assertTrue(mountTable.isReadOnly());
assertEquals(testOwner, mountTable.getOwnerName());
assertEquals(testGroup, mountTable.getGroupName());
assertEquals((short)0455, mountTable.getMode().toShort());
}
@Test
public void testUpdateOrderMountTable() throws Exception {
testUpdateOrderMountTable(DestinationOrder.HASH);
testUpdateOrderMountTable(DestinationOrder.LOCAL);
testUpdateOrderMountTable(DestinationOrder.RANDOM);
testUpdateOrderMountTable(DestinationOrder.HASH_ALL);
}
private void testUpdateOrderMountTable(DestinationOrder order)
throws Exception {
// Add a mount table
String nsId = "ns0";
String src = "/test-updateOrderMountTable-"+order.toString();
String dest = "/UpdateOrderMountTable";
String[] argv = new String[] {"-add", src, nsId, dest};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
GetMountTableEntriesRequest getRequest =
GetMountTableEntriesRequest.newInstance(src);
GetMountTableEntriesResponse getResponse =
client.getMountTableManager().getMountTableEntries(getRequest);
// Ensure mount table added successfully
MountTable mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
assertEquals(DestinationOrder.HASH, mountTable.getDestOrder());
// Update the order
argv = new String[] {"-update", src, nsId, dest, "-order",
order.toString()};
assertEquals(0, ToolRunner.run(admin, argv));
stateStore.loadCache(MountTableStoreImpl.class, true);
getResponse = client.getMountTableManager()
.getMountTableEntries(getRequest);
// Ensure the destination updated successfully
mountTable = getResponse.getEntries().get(0);
assertEquals(src, mountTable.getSourcePath());
assertEquals(nsId, mountTable.getDestinations().get(0).getNameserviceId());
assertEquals(dest, mountTable.getDestinations().get(0).getDest());
assertEquals(order, mountTable.getDestOrder());
}
}

6
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
View File

@ -425,7 +425,8 @@ Runs the DFS router. See [Router](./HDFSRouterFederation.html#Router) for more i
Usage:
hdfs dfsrouteradmin
[-add <source> <nameservice> <destination> [-readonly] -owner <owner> -group <group> -mode <mode>]
[-add <source> <nameservice1, nameservice2, ...> <destination> [-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] -owner <owner> -group <group> -mode <mode>]
[-update <source> <nameservice1, nameservice2, ...> <destination> [-readonly] [-order HASH|LOCAL|RANDOM|HASH_ALL] -owner <owner> -group <group> -mode <mode>]
[-rm <source>]
[-ls <path>]
[-setQuota <path> -nsQuota <nsQuota> -ssQuota <quota in bytes or quota size string>]
@ -436,7 +437,8 @@ Usage:
| COMMAND\_OPTION | Description |
|:---- |:---- |
| `-add` *source* *nameservice* *destination* | Add a mount table entry or update if it exists. |
| `-add` *source* *nameservices* *destination* | Add a mount table entry or update if it exists. |
| `-update` *source* *nameservices* *destination* | Update a mount table entry or create one if it does not exist. |
| `-rm` *source* | Remove mount point of specified path. |
| `-ls` *path* | List mount points under specified path. |
| `-setQuota` *path* `-nsQuota` *nsQuota* `-ssQuota` *ssQuota* | Set quota for specified path. See [HDFS Quotas Guide](./HdfsQuotaAdminGuide.html) for the quota detail. |