HDFS-14188. Make hdfs ec -verifyClusterSetup command accept an erasure coding policy as a parameter. Contributed by Kitti Nanasi.
Signed-off-by: Wei-Chiu Chuang <weichiu@apache.org>
This commit is contained in:
parent
b8de78c570
commit
14282e311b
|
@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configured;
|
|||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.fs.RemoteIterator;
|
||||
import org.apache.hadoop.fs.shell.CommandFormat;
|
||||
import org.apache.hadoop.hdfs.DistributedFileSystem;
|
||||
import org.apache.hadoop.hdfs.protocol.AddErasureCodingPolicyResponse;
|
||||
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
|
||||
|
@ -612,24 +613,32 @@ public class ECAdmin extends Configured implements Tool {
|
|||
|
||||
@Override
|
||||
public String getShortUsage() {
|
||||
return "[" + getName() + "]\n";
|
||||
return "[" + getName() + " [-policy <policy>...<policy>]]\n";
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getLongUsage() {
|
||||
TableListing listing = AdminHelper.getOptionDescriptionListing();
|
||||
listing.addRow("<policy>", "The name of the erasure coding policy");
|
||||
return getShortUsage() + "\n"
|
||||
+ "Verify the cluster setup can support all enabled erasure coding"
|
||||
+ " policies.\n";
|
||||
+ "Verify if the cluster setup can support all enabled erasure " +
|
||||
"coding policies. If optional parameter -policy is specified, " +
|
||||
"verify if the cluster setup can support the given policy.\n";
|
||||
}
|
||||
|
||||
@Override
|
||||
public int run(Configuration conf, List<String> args) throws IOException {
|
||||
if (args.size() > 0) {
|
||||
System.err.println(getName() + ": Too many arguments");
|
||||
return 1;
|
||||
}
|
||||
boolean isPolicyOption = StringUtils.popOption("-policy", args);
|
||||
final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
|
||||
ECTopologyVerifierResult result = getECTopologyVerifierResult(dfs);
|
||||
ECTopologyVerifierResult result;
|
||||
if (isPolicyOption) {
|
||||
CommandFormat c = new CommandFormat(1, Integer.MAX_VALUE);
|
||||
c.parse(args);
|
||||
String[] parameters = args.toArray(new String[args.size()]);
|
||||
result = getECTopologyResultForPolicies(dfs, parameters);
|
||||
} else {
|
||||
result = getECTopologyVerifierResult(dfs);
|
||||
}
|
||||
System.out.println(result.getResultMessage());
|
||||
if (result.isSupported()) {
|
||||
return 0;
|
||||
|
@ -649,6 +658,21 @@ public class ECAdmin extends Configured implements Tool {
|
|||
getEnabledPolicies(policies));
|
||||
}
|
||||
|
||||
private static ECTopologyVerifierResult getECTopologyResultForPolicies(
|
||||
final DistributedFileSystem dfs, final String... policyNames)
|
||||
throws IOException {
|
||||
ErasureCodingPolicy[] policies =
|
||||
new ErasureCodingPolicy[policyNames.length];
|
||||
for (int i = 0; i < policyNames.length; i++) {
|
||||
policies[i] =
|
||||
getPolicy(dfs.getClient().getNamenode().getErasureCodingPolicies(),
|
||||
policyNames[i]);
|
||||
}
|
||||
final DatanodeInfo[] report = dfs.getClient().getNamenode()
|
||||
.getDatanodeReport(HdfsConstants.DatanodeReportType.ALL);
|
||||
return ECTopologyVerifier.getECTopologyVerifierResult(report, policies);
|
||||
}
|
||||
|
||||
private static ECTopologyVerifierResult getECTopologyVerifierResultForPolicy(
|
||||
final DistributedFileSystem dfs, final String policyName)
|
||||
throws IOException {
|
||||
|
|
|
@ -488,6 +488,7 @@ Usage:
|
|||
[-listCodecs]
|
||||
[-enablePolicy -policy <policyName>]
|
||||
[-disablePolicy -policy <policyName>]
|
||||
[-verifyClusterSetup -policy <policyName>...<policyName>]
|
||||
[-help [cmd ...]]
|
||||
|
||||
| COMMAND\_OPTION | Description |
|
||||
|
@ -500,6 +501,7 @@ Usage:
|
|||
|-listCodecs| Get the list of supported erasure coding codecs and coders in system|
|
||||
|-enablePolicy| Enable an ErasureCoding policy in system|
|
||||
|-disablePolicy| Disable an ErasureCoding policy in system|
|
||||
|-verifyClusterSetup| Verify if the cluster setup can support a list of erasure coding policies|
|
||||
|
||||
Runs the ErasureCoding CLI. See [HDFS ErasureCoding](./HDFSErasureCoding.html#Administrative_commands) for more information on this command.
|
||||
|
||||
|
|
|
@ -170,6 +170,7 @@ Deployment
|
|||
[-listCodecs]
|
||||
[-enablePolicy -policy <policyName>]
|
||||
[-disablePolicy -policy <policyName>]
|
||||
[-verifyClusterSetup -policy <policyName>...<policyName>]
|
||||
[-help [cmd ...]]
|
||||
|
||||
Below are the details about each command.
|
||||
|
@ -221,6 +222,10 @@ Below are the details about each command.
|
|||
|
||||
Disable an erasure coding policy.
|
||||
|
||||
* `[-verifyClusterSetup -policy <policyName>...<policyName>]`
|
||||
|
||||
Verify if the cluster setup can support all enabled erasure coding policies. If optional parameter -policy is specified, verify if the cluster setup can support the given policy or policies.
|
||||
|
||||
Limitations
|
||||
-----------
|
||||
|
||||
|
|
|
@ -68,8 +68,7 @@ public class TestECAdmin {
|
|||
try {
|
||||
System.out.flush();
|
||||
System.err.flush();
|
||||
out.reset();
|
||||
err.reset();
|
||||
resetOutputs();
|
||||
} finally {
|
||||
System.setOut(OLD_OUT);
|
||||
System.setErr(OLD_ERR);
|
||||
|
@ -242,4 +241,59 @@ public class TestECAdmin {
|
|||
err.toString().contains("RemoteException: The policy name " +
|
||||
"NonExistentPolicy does not exist"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testVerifyClusterSetupWithGivenPolicies() throws Exception {
|
||||
cluster = DFSTestUtil.setupCluster(conf, 5, 2, 0);
|
||||
|
||||
String[] args = new String[]{"-verifyClusterSetup", "-policy",
|
||||
"RS-3-2-1024k"};
|
||||
int ret = admin.run(args);
|
||||
LOG.info("Command stdout: {}", out.toString());
|
||||
LOG.info("Command stderr: {}", err.toString());
|
||||
assertEquals("Return value of the command is not successful", 2, ret);
|
||||
assertTrue("Result of cluster topology verify " +
|
||||
"should be logged correctly", out.toString()
|
||||
.contains("less than the minimum required number of racks (3) " +
|
||||
"for the erasure coding policies: RS-3-2-1024k"));
|
||||
assertTrue("Error output should be empty", err.toString().isEmpty());
|
||||
|
||||
resetOutputs();
|
||||
args = new String[]{"-verifyClusterSetup", "-policy",
|
||||
"RS-10-4-1024k", "RS-3-2-1024k"};
|
||||
ret = admin.run(args);
|
||||
LOG.info("Command stdout: {}", out.toString());
|
||||
LOG.info("Command stderr: {}", err.toString());
|
||||
assertEquals("Return value of the command is not successful", 2, ret);
|
||||
assertTrue("Result of cluster topology verify " +
|
||||
"should be logged correctly", out.toString()
|
||||
.contains(
|
||||
"for the erasure coding policies: RS-10-4-1024k, RS-3-2-1024k"));
|
||||
assertTrue("Error output should be empty", err.toString().isEmpty());
|
||||
|
||||
resetOutputs();
|
||||
args = new String[]{"-verifyClusterSetup", "-policy", "invalidPolicy"};
|
||||
ret = admin.run(args);
|
||||
LOG.info("Command stdout: {}", out.toString());
|
||||
LOG.info("Command stderr: {}", err.toString());
|
||||
assertEquals("Return value of the command is not successful", -1, ret);
|
||||
assertTrue("Error message should be logged", err.toString()
|
||||
.contains("The given erasure coding policy invalidPolicy " +
|
||||
"does not exist."));
|
||||
|
||||
resetOutputs();
|
||||
args = new String[]{"-verifyClusterSetup", "-policy"};
|
||||
ret = admin.run(args);
|
||||
LOG.info("Command stdout: {}", out.toString());
|
||||
LOG.info("Command stderr: {}", err.toString());
|
||||
assertEquals("Return value of the command is not successful", -1, ret);
|
||||
assertTrue("Error message should be logged", err.toString()
|
||||
.contains("NotEnoughArgumentsException: Not enough arguments: " +
|
||||
"expected 1 but got 0"));
|
||||
}
|
||||
|
||||
private void resetOutputs() {
|
||||
out.reset();
|
||||
err.reset();
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue