HDFS-10557. Fix handling of the -fs Generic option. (Arpit Agarwal)
This commit is contained in:
parent
7b23ad1ef7
commit
66fa34c839
|
@ -45,9 +45,7 @@ import org.slf4j.LoggerFactory;
|
|||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.MalformedURLException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.net.URL;
|
||||
import java.nio.charset.Charset;
|
||||
import java.nio.file.Files;
|
||||
|
@ -93,8 +91,7 @@ public abstract class Command extends Configured {
|
|||
* Executes the Client Calls.
|
||||
*
|
||||
* @param cmd - CommandLine
|
||||
* @throws IOException
|
||||
* @throws URISyntaxException
|
||||
* @throws Exception
|
||||
*/
|
||||
public abstract void execute(CommandLine cmd) throws Exception;
|
||||
|
||||
|
@ -103,22 +100,6 @@ public abstract class Command extends Configured {
|
|||
*/
|
||||
public abstract void printHelp();
|
||||
|
||||
/**
|
||||
* verifies user provided URL.
|
||||
*
|
||||
* @param uri - UrlString
|
||||
* @return URL
|
||||
* @throws URISyntaxException, MalformedURLException
|
||||
*/
|
||||
protected URI verifyURI(String uri)
|
||||
throws URISyntaxException, MalformedURLException {
|
||||
if ((uri == null) || uri.isEmpty()) {
|
||||
throw new MalformedURLException(
|
||||
"A valid URI is needed to execute this command.");
|
||||
}
|
||||
return new URI(uri);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process the URI and return the cluster with nodes setup. This is used in
|
||||
* all commands.
|
||||
|
@ -130,11 +111,8 @@ public abstract class Command extends Configured {
|
|||
protected DiskBalancerCluster readClusterInfo(CommandLine cmd) throws
|
||||
Exception {
|
||||
Preconditions.checkNotNull(cmd);
|
||||
Preconditions
|
||||
.checkState(cmd.getOptionValue(DiskBalancer.NAMENODEURI) != null,
|
||||
"Required argument missing : uri");
|
||||
|
||||
setClusterURI(verifyURI(cmd.getOptionValue(DiskBalancer.NAMENODEURI)));
|
||||
setClusterURI(FileSystem.getDefaultUri(getConf()));
|
||||
LOG.debug("using name node URI : {}", this.getClusterURI());
|
||||
ClusterConnector connector = ConnectorFactory.getCluster(this.clusterURI,
|
||||
getConf());
|
||||
|
@ -346,6 +324,7 @@ public abstract class Command extends Configured {
|
|||
*
|
||||
* @param fileName - fileName to open.
|
||||
* @return OutputStream.
|
||||
* @throws IOException
|
||||
*/
|
||||
protected FSDataOutputStream create(String fileName) throws IOException {
|
||||
Preconditions.checkNotNull(fileName);
|
||||
|
|
|
@ -63,10 +63,9 @@ public class PlanCommand extends Command {
|
|||
this.thresholdPercentage = 1;
|
||||
this.bandwidth = 0;
|
||||
this.maxError = 0;
|
||||
addValidCommandParameters(DiskBalancer.NAMENODEURI, "Name Node URI or " +
|
||||
"file URI for cluster");
|
||||
|
||||
addValidCommandParameters(DiskBalancer.OUTFILE, "Output file");
|
||||
addValidCommandParameters(DiskBalancer.OUTFILE, "Output directory in " +
|
||||
"HDFS. The generated plan will be written to a file in this " +
|
||||
"directory.");
|
||||
addValidCommandParameters(DiskBalancer.BANDWIDTH, "Maximum Bandwidth to " +
|
||||
"be used while copying.");
|
||||
addValidCommandParameters(DiskBalancer.THRESHOLD, "Percentage skew that " +
|
||||
|
@ -188,7 +187,7 @@ public class PlanCommand extends Command {
|
|||
*/
|
||||
@Override
|
||||
public void printHelp() {
|
||||
String header = "creates a plan that describes how much data should be " +
|
||||
String header = "Creates a plan that describes how much data should be " +
|
||||
"moved between disks.\n\n";
|
||||
|
||||
String footer = "\nPlan command creates a set of steps that represent a " +
|
||||
|
@ -196,7 +195,7 @@ public class PlanCommand extends Command {
|
|||
" will balance the data.";
|
||||
|
||||
HelpFormatter helpFormatter = new HelpFormatter();
|
||||
helpFormatter.printHelp("hdfs diskbalancer -uri <namenode> -plan " +
|
||||
helpFormatter.printHelp("hdfs diskbalancer -plan " +
|
||||
"<hostname> [options]", header, DiskBalancer.getPlanOptions(), footer);
|
||||
}
|
||||
|
||||
|
|
|
@ -201,9 +201,9 @@ public class ReportCommand extends Command {
|
|||
" datanode, or prints out the list of nodes that will benefit from " +
|
||||
"running disk balancer. Top defaults to " + getDefaultTop();
|
||||
String footer = ". E.g.:\n"
|
||||
+ "hdfs diskbalancer -fs http://namenode.uri -report\n"
|
||||
+ "hdfs diskbalancer -fs http://namenode.uri -report -top 5\n"
|
||||
+ "hdfs diskbalancer -fs http://namenode.uri -report "
|
||||
+ "hdfs diskbalancer -report\n"
|
||||
+ "hdfs diskbalancer -report -top 5\n"
|
||||
+ "hdfs diskbalancer -report "
|
||||
+ "-node {DataNodeID | IP | Hostname}";
|
||||
|
||||
HelpFormatter helpFormatter = new HelpFormatter();
|
||||
|
|
|
@ -67,7 +67,8 @@ public class GreedyPlanner implements Planner {
|
|||
long startTime = Time.monotonicNow();
|
||||
NodePlan plan = new NodePlan(node.getDataNodeName(),
|
||||
node.getDataNodePort());
|
||||
LOG.info("Starting plan for Node : " + node.getDataNodeUUID());
|
||||
LOG.info("Starting plan for Node : {}:{}",
|
||||
node.getDataNodeName(), node.getDataNodePort());
|
||||
while (node.isBalancingNeeded(this.threshold)) {
|
||||
for (DiskBalancerVolumeSet vSet : node.getVolumeSets().values()) {
|
||||
balanceVolumeSet(node, vSet, plan);
|
||||
|
@ -76,8 +77,9 @@ public class GreedyPlanner implements Planner {
|
|||
|
||||
long endTime = Time.monotonicNow();
|
||||
String message = String
|
||||
.format("Compute Plan for Node : %s took %d ms ",
|
||||
node.getDataNodeUUID(), endTime - startTime);
|
||||
.format("Compute Plan for Node : %s:%d took %d ms ",
|
||||
node.getDataNodeName(), node.getDataNodePort(),
|
||||
endTime - startTime);
|
||||
LOG.info(message);
|
||||
return plan;
|
||||
}
|
||||
|
|
|
@ -51,16 +51,6 @@ import java.io.PrintStream;
|
|||
* utilization equal and then those moves are executed by the datanode.
|
||||
*/
|
||||
public class DiskBalancer extends Configured implements Tool {
|
||||
/**
|
||||
* NameNodeURI can point to either a real namenode, or a json file that
|
||||
* contains the diskBalancer data in json form, that jsonNodeConnector knows
|
||||
* how to deserialize.
|
||||
* <p>
|
||||
* Expected formats are :
|
||||
* <p>
|
||||
* hdfs://namenode.uri or file:///data/myCluster.json
|
||||
*/
|
||||
public static final String NAMENODEURI = "fs";
|
||||
/**
|
||||
* Computes a plan for a given set of nodes.
|
||||
*/
|
||||
|
@ -275,13 +265,6 @@ public class DiskBalancer extends Configured implements Tool {
|
|||
*/
|
||||
private void addPlanCommands(Options opt) {
|
||||
|
||||
Option uri = OptionBuilder.withLongOpt(NAMENODEURI)
|
||||
.withDescription("Address of the Namenode. e,g. hdfs://namenode")
|
||||
.hasArg()
|
||||
.create();
|
||||
getPlanOptions().addOption(uri);
|
||||
opt.addOption(uri);
|
||||
|
||||
Option plan = OptionBuilder.withLongOpt(PLAN)
|
||||
.withDescription("creates a plan for datanode.")
|
||||
.hasArg()
|
||||
|
@ -336,7 +319,6 @@ public class DiskBalancer extends Configured implements Tool {
|
|||
private void addHelpCommands(Options opt) {
|
||||
Option help = OptionBuilder.withLongOpt(HELP)
|
||||
.hasOptionalArg()
|
||||
.withArgName(HELP)
|
||||
.withDescription("valid commands are plan | execute | query | cancel" +
|
||||
" | report")
|
||||
.create();
|
||||
|
|
|
@ -53,11 +53,9 @@ The following sections discusses what commands are supported by disk balancer
|
|||
|
||||
The plan command can be run against a given datanode by running
|
||||
|
||||
`hdfs diskbalancer -uri hdfs://mycluster.com -plan node1.mycluster.com`
|
||||
`hdfs diskbalancer -plan node1.mycluster.com`
|
||||
|
||||
uri is the address of the namenode and -plan points to the datanode that we
|
||||
need to plan for. By deafult, plan command writes its output to
|
||||
**/system/diskbalancer**.
|
||||
The command accepts [Generic Options](../hadoop-common/CommandsManual.html#Generic_Options).
|
||||
|
||||
The plan command also has a set of parameters that allows user to control
|
||||
the output and execution of the plan.
|
||||
|
|
|
@ -31,6 +31,7 @@ import java.util.Scanner;
|
|||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.hdfs.DFSConfigKeys;
|
||||
import org.apache.hadoop.hdfs.HdfsConfiguration;
|
||||
import org.apache.hadoop.hdfs.MiniDFSCluster;
|
||||
|
@ -74,8 +75,7 @@ public class TestDiskBalancerCommand {
|
|||
/* test basic report */
|
||||
@Test(timeout=60000)
|
||||
public void testReportSimple() throws Exception {
|
||||
final String cmdLine = String.format("hdfs diskbalancer -fs %s -report",
|
||||
clusterJson.toString());
|
||||
final String cmdLine = "hdfs diskbalancer -report";
|
||||
final List<String> outputs = runCommand(cmdLine);
|
||||
|
||||
assertThat(
|
||||
|
@ -103,8 +103,7 @@ public class TestDiskBalancerCommand {
|
|||
/* test less than 64 DataNode(s) as total, e.g., -report -top 32 */
|
||||
@Test(timeout=60000)
|
||||
public void testReportLessThanTotal() throws Exception {
|
||||
final String cmdLine = String.format(
|
||||
"hdfs diskbalancer -fs %s -report -top 32", clusterJson.toString());
|
||||
final String cmdLine = "hdfs diskbalancer -report -top 32";
|
||||
final List<String> outputs = runCommand(cmdLine);
|
||||
|
||||
assertThat(
|
||||
|
@ -127,8 +126,7 @@ public class TestDiskBalancerCommand {
|
|||
/* test more than 64 DataNode(s) as total, e.g., -report -top 128 */
|
||||
@Test(timeout=60000)
|
||||
public void testReportMoreThanTotal() throws Exception {
|
||||
final String cmdLine = String.format(
|
||||
"hdfs diskbalancer -fs %s -report -top 128", clusterJson.toString());
|
||||
final String cmdLine = "hdfs diskbalancer -report -top 128";
|
||||
final List<String> outputs = runCommand(cmdLine);
|
||||
|
||||
assertThat(
|
||||
|
@ -152,8 +150,7 @@ public class TestDiskBalancerCommand {
|
|||
/* test invalid top limit, e.g., -report -top xx */
|
||||
@Test(timeout=60000)
|
||||
public void testReportInvalidTopLimit() throws Exception {
|
||||
final String cmdLine = String.format(
|
||||
"hdfs diskbalancer -fs %s -report -top xx", clusterJson.toString());
|
||||
final String cmdLine = "hdfs diskbalancer -report -top xx";
|
||||
final List<String> outputs = runCommand(cmdLine);
|
||||
|
||||
assertThat(
|
||||
|
@ -177,14 +174,11 @@ public class TestDiskBalancerCommand {
|
|||
containsString("9 volumes with node data density 1.97"))));
|
||||
}
|
||||
|
||||
/* test -report -node DataNodeID */
|
||||
@Test(timeout=60000)
|
||||
public void testReportNode() throws Exception {
|
||||
final String cmdLine = String
|
||||
.format(
|
||||
"hdfs diskbalancer -fs %s -report -node "
|
||||
+ "a87654a9-54c7-4693-8dd9-c9c7021dc340",
|
||||
clusterJson.toString());
|
||||
final String cmdLine =
|
||||
"hdfs diskbalancer -report -node " +
|
||||
"a87654a9-54c7-4693-8dd9-c9c7021dc340";
|
||||
final List<String> outputs = runCommand(cmdLine);
|
||||
|
||||
assertThat(
|
||||
|
@ -275,6 +269,7 @@ public class TestDiskBalancerCommand {
|
|||
org.apache.hadoop.hdfs.tools.DiskBalancer db =
|
||||
new org.apache.hadoop.hdfs.tools.DiskBalancer(conf);
|
||||
|
||||
FileSystem.setDefaultUri(conf, clusterJson);
|
||||
ByteArrayOutputStream bufOut = new ByteArrayOutputStream();
|
||||
PrintStream out = new PrintStream(bufOut);
|
||||
db.run(cmds, out);
|
||||
|
|
Loading…
Reference in New Issue