HBASE-16982 Better integrate Apache CLI in AbstractHBaseTool.
- processOldArgs() in AbstractHBaseTool will make it possible to change existing tools to use Apache CLI in a backward compatible manner. - Changes ExportSnapshot tool as a proof-of-concept - Update version of Apache CLI from 1.2 to 1.3.1 Change-Id: I4bb2e7da4f2ed0e12e28621ab16459309fa36d03
This commit is contained in:
Apekshit Sharma
parent
9afcb4366d
commit
52241c90e4
|
|
@ -17,15 +17,21 @@
|
|||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.cli.BasicParser;
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.DefaultParser;
|
||||
import org.apache.commons.cli.HelpFormatter;
|
||||
import org.apache.commons.cli.MissingOptionException;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.commons.cli.Options;
|
||||
import org.apache.commons.cli.ParseException;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.conf.Configurable;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
|
|
@ -37,7 +43,7 @@ import org.apache.hadoop.util.ToolRunner;
|
|||
* command-line argument parsing.
|
||||
*/
|
||||
@InterfaceAudience.Private
|
||||
public abstract class AbstractHBaseTool implements Tool {
|
||||
public abstract class AbstractHBaseTool implements Tool, Configurable {
|
||||
protected static final int EXIT_SUCCESS = 0;
|
||||
protected static final int EXIT_FAILURE = 1;
|
||||
|
||||
|
|
@ -52,6 +58,17 @@ public abstract class AbstractHBaseTool implements Tool {
|
|||
|
||||
protected String[] cmdLineArgs = null;
|
||||
|
||||
// To print options in order they were added in help text.
|
||||
private HashMap<Option, Integer> optionsOrder = new HashMap<>();
|
||||
private int optionsCount = 0;
|
||||
|
||||
private class OptionsOrderComparator implements Comparator<Option> {
|
||||
@Override
|
||||
public int compare(Option o1, Option o2) {
|
||||
return optionsOrder.get(o1) - optionsOrder.get(o2);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Override this to add command-line options using {@link #addOptWithArg}
|
||||
* and similar methods.
|
||||
|
|
@ -66,6 +83,20 @@ public abstract class AbstractHBaseTool implements Tool {
|
|||
/** The "main function" of the tool */
|
||||
protected abstract int doWork() throws Exception;
|
||||
|
||||
/**
|
||||
* For backward compatibility. DO NOT use it for new tools.
|
||||
* We have options in existing tools which can't be ported to Apache CLI's {@link Option}.
|
||||
* (because they don't pass validation, for e.g. "-copy-to". "-" means short name
|
||||
* which doesn't allow '-' in name). This function is to allow tools to have, for time being,
|
||||
* parameters which can't be parsed using {@link Option}.
|
||||
* Overrides should consume all valid legacy arguments. If the param 'args' is not empty on
|
||||
* return, it means there were invalid options, in which case we'll exit from the tool.
|
||||
* Note that it's called before {@link #processOptions(CommandLine)}, which means new options'
|
||||
* values will override old ones'.
|
||||
*/
|
||||
protected void processOldArgs(List<String> args) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Configuration getConf() {
|
||||
return conf;
|
||||
|
|
@ -85,17 +116,29 @@ public abstract class AbstractHBaseTool implements Tool {
|
|||
}
|
||||
|
||||
CommandLine cmd;
|
||||
List<String> argsList = new ArrayList<>();
|
||||
for (String arg : args) {
|
||||
argsList.add(arg);
|
||||
}
|
||||
// For backward compatibility of args which can't be parsed as Option. See javadoc for
|
||||
// processOldArgs(..)
|
||||
processOldArgs(argsList);
|
||||
try {
|
||||
addOptions();
|
||||
if (isHelpCommand(args)) {
|
||||
printUsage();
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
// parse the command line arguments
|
||||
cmd = new BasicParser().parse(options, args);
|
||||
String[] remainingArgs = new String[argsList.size()];
|
||||
argsList.toArray(remainingArgs);
|
||||
cmd = new DefaultParser().parse(options, remainingArgs);
|
||||
} catch (MissingOptionException e) {
|
||||
LOG.error(e.getMessage());
|
||||
LOG.error("Use -h or --help for usage instructions.");
|
||||
return EXIT_FAILURE;
|
||||
} catch (ParseException e) {
|
||||
LOG.error("Error when parsing command-line arguments", e);
|
||||
printUsage();
|
||||
LOG.error("Use -h or --help for usage instructions.");
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
|
|
@ -114,7 +157,7 @@ public abstract class AbstractHBaseTool implements Tool {
|
|||
private boolean isHelpCommand(String[] args) throws ParseException {
|
||||
Options helpOption = new Options().addOption(HELP_OPTION);
|
||||
// this parses the command line but doesn't throw an exception on unknown options
|
||||
CommandLine cl = new BasicParser().parse(helpOption, args, true);
|
||||
CommandLine cl = new DefaultParser().parse(helpOption, args, true);
|
||||
return cl.getOptions().length != 0;
|
||||
}
|
||||
|
||||
|
|
@ -126,39 +169,46 @@ public abstract class AbstractHBaseTool implements Tool {
|
|||
final String usageFooter) {
|
||||
HelpFormatter helpFormatter = new HelpFormatter();
|
||||
helpFormatter.setWidth(120);
|
||||
helpFormatter.setOptionComparator(new OptionsOrderComparator());
|
||||
helpFormatter.printHelp(usageStr, usageHeader, options, usageFooter);
|
||||
}
|
||||
|
||||
protected void addOption(Option option) {
|
||||
options.addOption(option);
|
||||
optionsOrder.put(option, optionsCount++);
|
||||
}
|
||||
|
||||
protected void addRequiredOption(Option option) {
|
||||
option.setRequired(true);
|
||||
addOption(option);
|
||||
}
|
||||
|
||||
protected void addRequiredOptWithArg(String opt, String description) {
|
||||
Option option = new Option(opt, true, description);
|
||||
option.setRequired(true);
|
||||
options.addOption(option);
|
||||
addOption(option);
|
||||
}
|
||||
|
||||
protected void addRequiredOptWithArg(String shortOpt, String longOpt, String description) {
|
||||
Option option = new Option(shortOpt, longOpt, true, description);
|
||||
option.setRequired(true);
|
||||
options.addOption(option);
|
||||
addOption(option);
|
||||
}
|
||||
|
||||
protected void addOptNoArg(String opt, String description) {
|
||||
options.addOption(opt, false, description);
|
||||
addOption(new Option(opt, false, description));
|
||||
}
|
||||
|
||||
protected void addOptNoArg(String shortOpt, String longOpt, String description) {
|
||||
options.addOption(shortOpt, longOpt, false, description);
|
||||
addOption(new Option(shortOpt, longOpt, false, description));
|
||||
}
|
||||
|
||||
protected void addOptWithArg(String opt, String description) {
|
||||
options.addOption(opt, true, description);
|
||||
addOption(new Option(opt, true, description));
|
||||
}
|
||||
|
||||
protected void addOptWithArg(String shortOpt, String longOpt, String description) {
|
||||
options.addOption(shortOpt, longOpt, true, description);
|
||||
addOption(new Option(shortOpt, longOpt, true, description));
|
||||
}
|
||||
|
||||
public int getOptionAsInt(CommandLine cmd, String opt, int defaultValue) {
|
||||
|
|
@ -176,6 +226,7 @@ public abstract class AbstractHBaseTool implements Tool {
|
|||
return defaultValue;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a number and enforce a range.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -0,0 +1,150 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with this
|
||||
* work for additional information regarding copyright ownership. The ASF
|
||||
* licenses this file to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
* License for the specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.util;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.apache.hadoop.hbase.util.AbstractHBaseTool.EXIT_FAILURE;
|
||||
import static org.apache.hadoop.hbase.util.AbstractHBaseTool.EXIT_SUCCESS;
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
public class AbstractHBaseToolTest {
|
||||
static final class Options {
|
||||
static final Option REQUIRED = new Option(null, "required", true, "");
|
||||
static final Option OPTIONAL = new Option(null, "optional", true, "");
|
||||
static final Option BOOLEAN = new Option(null, "boolean", false, "");
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple tool to test options parsing.
|
||||
* 3 options: required, optional, and boolean
|
||||
* 2 deprecated options to test backward compatibility: -opt (old version of --optional) and
|
||||
* -bool (old version of --boolean).
|
||||
*/
|
||||
private class TestTool extends AbstractHBaseTool {
|
||||
String requiredValue;
|
||||
String optionalValue;
|
||||
boolean booleanValue;
|
||||
|
||||
@Override
|
||||
protected void addOptions() {
|
||||
addRequiredOption(Options.REQUIRED);
|
||||
addOption(Options.OPTIONAL);
|
||||
addOption(Options.BOOLEAN);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processOptions(CommandLine cmd) {
|
||||
requiredValue = cmd.getOptionValue(Options.REQUIRED.getLongOpt());
|
||||
if (cmd.hasOption(Options.OPTIONAL.getLongOpt())) {
|
||||
optionalValue = cmd.getOptionValue(Options.OPTIONAL.getLongOpt());
|
||||
}
|
||||
booleanValue = booleanValue || cmd.hasOption(Options.BOOLEAN.getLongOpt());
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void processOldArgs(List<String> args) {
|
||||
List<String> invalidArgs = new ArrayList<>();
|
||||
while(args.size() > 0) {
|
||||
String cmd = args.remove(0);
|
||||
if (cmd.equals("-opt")) {
|
||||
optionalValue = args.remove(0);
|
||||
} else if (cmd.equals("-bool")) {
|
||||
booleanValue = true;
|
||||
} else {
|
||||
invalidArgs.add(cmd);
|
||||
}
|
||||
}
|
||||
args.addAll(invalidArgs);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected int doWork() throws Exception {
|
||||
return EXIT_SUCCESS;
|
||||
}
|
||||
}
|
||||
|
||||
TestTool tool;
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
tool = new TestTool();
|
||||
tool.setConf(HBaseConfiguration.create());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAllOptionsSet() throws Exception {
|
||||
String[] args = new String[] { "--required=foo", "--optional=bar", "--boolean"};
|
||||
int returnValue = tool.run(args);
|
||||
assertEquals(EXIT_SUCCESS, returnValue);
|
||||
assertEquals("foo", tool.requiredValue);
|
||||
assertEquals("bar", tool.optionalValue);
|
||||
assertTrue(tool.booleanValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOptionsNotSet() throws Exception {
|
||||
String[] args = new String[] { "--required=foo" };
|
||||
int returnValue = tool.run(args);
|
||||
assertEquals(EXIT_SUCCESS, returnValue);
|
||||
assertEquals("foo", tool.requiredValue);
|
||||
assertNull(tool.optionalValue);
|
||||
assertFalse(tool.booleanValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testMissingRequiredOption() throws Exception {
|
||||
String[] args = new String[0];
|
||||
int returnValue = tool.run(args);
|
||||
assertEquals(EXIT_FAILURE, returnValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFailureOnUnrecognizedOption() throws Exception {
|
||||
String[] args = new String[] { "--required=foo", "-asdfs" };
|
||||
int returnValue = tool.run(args);
|
||||
assertEquals(EXIT_FAILURE, returnValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testOldOptionsWork() throws Exception {
|
||||
String[] args = new String[] { "--required=foo", "-opt", "bar", "-bool" };
|
||||
int returnValue = tool.run(args);
|
||||
assertEquals(EXIT_SUCCESS, returnValue);
|
||||
assertEquals("foo", tool.requiredValue);
|
||||
assertEquals("bar", tool.optionalValue);
|
||||
assertTrue(tool.booleanValue);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testNewOptionOverridesOldOption() throws Exception {
|
||||
String[] args = new String[] { "--required=foo", "--optional=baz", "-opt", "bar", "-bool" };
|
||||
int returnValue = tool.run(args);
|
||||
assertEquals(EXIT_SUCCESS, returnValue);
|
||||
assertEquals("foo", tool.requiredValue);
|
||||
assertEquals("baz", tool.optionalValue);
|
||||
assertTrue(tool.booleanValue);
|
||||
}
|
||||
}
|
||||
|
|
@ -31,12 +31,13 @@ import java.util.LinkedList;
|
|||
import java.util.List;
|
||||
import java.util.Random;
|
||||
|
||||
import org.apache.commons.cli.CommandLine;
|
||||
import org.apache.commons.cli.Option;
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceAudience;
|
||||
import org.apache.hadoop.hbase.classification.InterfaceStability;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.conf.Configured;
|
||||
import org.apache.hadoop.fs.FSDataInputStream;
|
||||
import org.apache.hadoop.fs.FSDataOutputStream;
|
||||
import org.apache.hadoop.fs.FileChecksum;
|
||||
|
|
@ -57,6 +58,7 @@ import org.apache.hadoop.hbase.mob.MobUtils;
|
|||
import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.SnapshotDescription;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
|
||||
import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
|
||||
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
|
||||
import org.apache.hadoop.hbase.util.FSUtils;
|
||||
import org.apache.hadoop.hbase.util.HFileArchiveUtil;
|
||||
import org.apache.hadoop.hbase.util.Pair;
|
||||
|
|
@ -76,7 +78,6 @@ import org.apache.hadoop.mapreduce.security.TokenCache;
|
|||
import org.apache.hadoop.hbase.io.hadoopbackport.ThrottledInputStream;
|
||||
import org.apache.hadoop.util.StringUtils;
|
||||
import org.apache.hadoop.util.Tool;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
|
||||
/**
|
||||
* Export the specified snapshot to a given FileSystem.
|
||||
|
|
@ -87,7 +88,7 @@ import org.apache.hadoop.util.ToolRunner;
|
|||
*/
|
||||
@InterfaceAudience.Public
|
||||
@InterfaceStability.Evolving
|
||||
public class ExportSnapshot extends Configured implements Tool {
|
||||
public class ExportSnapshot extends AbstractHBaseTool implements Tool {
|
||||
public static final String NAME = "exportsnapshot";
|
||||
/** Configuration prefix for overrides for the source filesystem */
|
||||
public static final String CONF_SOURCE_PREFIX = NAME + ".from.";
|
||||
|
|
@ -114,6 +115,34 @@ public class ExportSnapshot extends Configured implements Tool {
|
|||
static final String CONF_TEST_FAILURE = "test.snapshot.export.failure";
|
||||
static final String CONF_TEST_RETRY = "test.snapshot.export.failure.retry";
|
||||
|
||||
|
||||
// Command line options and defaults.
|
||||
static final class Options {
|
||||
static final Option SNAPSHOT = new Option(null, "snapshot", true, "Snapshot to restore.");
|
||||
static final Option TARGET_NAME = new Option(null, "target", true,
|
||||
"Target name for the snapshot.");
|
||||
static final Option COPY_TO = new Option(null, "copy-to", true, "Remote "
|
||||
+ "destination hdfs://");
|
||||
static final Option COPY_FROM = new Option(null, "copy-from", true,
|
||||
"Input folder hdfs:// (default hbase.rootdir)");
|
||||
static final Option NO_CHECKSUM_VERIFY = new Option(null, "no-checksum-verify", false,
|
||||
"Do not verify checksum, use name+length only.");
|
||||
static final Option NO_TARGET_VERIFY = new Option(null, "no-target-verify", false,
|
||||
"Do not verify the integrity of the exported snapshot.");
|
||||
static final Option OVERWRITE = new Option(null, "overwrite", false,
|
||||
"Rewrite the snapshot manifest if already exists.");
|
||||
static final Option CHUSER = new Option(null, "chuser", true,
|
||||
"Change the owner of the files to the specified one.");
|
||||
static final Option CHGROUP = new Option(null, "chgroup", true,
|
||||
"Change the group of the files to the specified one.");
|
||||
static final Option CHMOD = new Option(null, "chmod", true,
|
||||
"Change the permission of the files to the specified one.");
|
||||
static final Option MAPPERS = new Option(null, "mappers", true,
|
||||
"Number of mappers to use during the copy (mapreduce.job.maps).");
|
||||
static final Option BANDWIDTH = new Option(null, "bandwidth", true,
|
||||
"Limit bandwidth to this value in MB/second.");
|
||||
}
|
||||
|
||||
// Export Map-Reduce Counters, to keep track of the progress
|
||||
public enum Counter {
|
||||
MISSING_FILES, FILES_COPIED, FILES_SKIPPED, COPY_FAILED,
|
||||
|
|
@ -851,77 +880,70 @@ public class ExportSnapshot extends Configured implements Tool {
|
|||
}
|
||||
}
|
||||
|
||||
private boolean verifyTarget = true;
|
||||
private boolean verifyChecksum = true;
|
||||
private String snapshotName = null;
|
||||
private String targetName = null;
|
||||
private boolean overwrite = false;
|
||||
private String filesGroup = null;
|
||||
private String filesUser = null;
|
||||
private Path outputRoot = null;
|
||||
private Path inputRoot = null;
|
||||
private int bandwidthMB = Integer.MAX_VALUE;
|
||||
private int filesMode = 0;
|
||||
private int mappers = 0;
|
||||
|
||||
@Override
|
||||
protected void processOptions(CommandLine cmd) {
|
||||
snapshotName = cmd.getOptionValue(Options.SNAPSHOT.getLongOpt(), snapshotName);
|
||||
targetName = cmd.getOptionValue(Options.TARGET_NAME.getLongOpt(), targetName);
|
||||
if (cmd.hasOption(Options.COPY_TO.getLongOpt())) {
|
||||
outputRoot = new Path(cmd.getOptionValue(Options.COPY_TO.getLongOpt()));
|
||||
}
|
||||
if (cmd.hasOption(Options.COPY_FROM.getLongOpt())) {
|
||||
inputRoot = new Path(cmd.getOptionValue(Options.COPY_FROM.getLongOpt()));
|
||||
}
|
||||
mappers = getOptionAsInt(cmd, Options.MAPPERS.getLongOpt(), mappers);
|
||||
filesUser = cmd.getOptionValue(Options.CHUSER.getLongOpt(), filesUser);
|
||||
filesGroup = cmd.getOptionValue(Options.CHGROUP.getLongOpt(), filesGroup);
|
||||
filesMode = getOptionAsInt(cmd, Options.CHMOD.getLongOpt(), filesMode);
|
||||
bandwidthMB = getOptionAsInt(cmd, Options.BANDWIDTH.getLongOpt(), bandwidthMB);
|
||||
overwrite = cmd.hasOption(Options.OVERWRITE.getLongOpt());
|
||||
// And verifyChecksum and verifyTarget with values read from old args in processOldArgs(...).
|
||||
verifyChecksum = !cmd.hasOption(Options.NO_CHECKSUM_VERIFY.getLongOpt());
|
||||
verifyTarget = !cmd.hasOption(Options.NO_TARGET_VERIFY.getLongOpt());
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the export snapshot by copying the snapshot metadata, hfiles and wals.
|
||||
* @return 0 on success, and != 0 upon failure.
|
||||
*/
|
||||
@Override
|
||||
public int run(String[] args) throws IOException {
|
||||
boolean verifyTarget = true;
|
||||
boolean verifyChecksum = true;
|
||||
String snapshotName = null;
|
||||
String targetName = null;
|
||||
boolean overwrite = false;
|
||||
String filesGroup = null;
|
||||
String filesUser = null;
|
||||
Path outputRoot = null;
|
||||
int bandwidthMB = Integer.MAX_VALUE;
|
||||
int filesMode = 0;
|
||||
int mappers = 0;
|
||||
|
||||
public int doWork() throws IOException {
|
||||
Configuration conf = getConf();
|
||||
Path inputRoot = FSUtils.getRootDir(conf);
|
||||
|
||||
// Process command line args
|
||||
for (int i = 0; i < args.length; i++) {
|
||||
String cmd = args[i];
|
||||
if (cmd.equals("-snapshot")) {
|
||||
snapshotName = args[++i];
|
||||
} else if (cmd.equals("-target")) {
|
||||
targetName = args[++i];
|
||||
} else if (cmd.equals("-copy-to")) {
|
||||
outputRoot = new Path(args[++i]);
|
||||
} else if (cmd.equals("-copy-from")) {
|
||||
inputRoot = new Path(args[++i]);
|
||||
FSUtils.setRootDir(conf, inputRoot);
|
||||
} else if (cmd.equals("-no-checksum-verify")) {
|
||||
verifyChecksum = false;
|
||||
} else if (cmd.equals("-no-target-verify")) {
|
||||
verifyTarget = false;
|
||||
} else if (cmd.equals("-mappers")) {
|
||||
mappers = Integer.parseInt(args[++i]);
|
||||
} else if (cmd.equals("-chuser")) {
|
||||
filesUser = args[++i];
|
||||
} else if (cmd.equals("-chgroup")) {
|
||||
filesGroup = args[++i];
|
||||
} else if (cmd.equals("-bandwidth")) {
|
||||
bandwidthMB = Integer.parseInt(args[++i]);
|
||||
} else if (cmd.equals("-chmod")) {
|
||||
filesMode = Integer.parseInt(args[++i], 8);
|
||||
} else if (cmd.equals("-overwrite")) {
|
||||
overwrite = true;
|
||||
} else if (cmd.equals("-h") || cmd.equals("--help")) {
|
||||
printUsageAndExit();
|
||||
} else {
|
||||
System.err.println("UNEXPECTED: " + cmd);
|
||||
printUsageAndExit();
|
||||
}
|
||||
}
|
||||
|
||||
// Check user options
|
||||
if (snapshotName == null) {
|
||||
System.err.println("Snapshot name not provided.");
|
||||
printUsageAndExit();
|
||||
LOG.error("Use -h or --help for usage instructions.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (outputRoot == null) {
|
||||
System.err.println("Destination file-system not provided.");
|
||||
printUsageAndExit();
|
||||
System.err.println("Destination file-system (--" + Options.COPY_TO.getLongOpt()
|
||||
+ ") not provided.");
|
||||
LOG.error("Use -h or --help for usage instructions.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (targetName == null) {
|
||||
targetName = snapshotName;
|
||||
}
|
||||
if (inputRoot == null) {
|
||||
inputRoot = FSUtils.getRootDir(conf);
|
||||
} else {
|
||||
FSUtils.setRootDir(conf, inputRoot);
|
||||
}
|
||||
|
||||
Configuration srcConf = HBaseConfiguration.createClusterConf(conf, null, CONF_SOURCE_PREFIX);
|
||||
srcConf.setBoolean("fs." + inputRoot.toUri().getScheme() + ".impl.disable.cache", true);
|
||||
|
|
@ -1053,51 +1075,37 @@ public class ExportSnapshot extends Configured implements Tool {
|
|||
}
|
||||
}
|
||||
|
||||
// ExportSnapshot
|
||||
private void printUsageAndExit() {
|
||||
System.err.printf("Usage: bin/hbase %s [options]%n", getClass().getName());
|
||||
System.err.println(" where [options] are:");
|
||||
System.err.println(" -h|-help Show this help and exit.");
|
||||
System.err.println(" -snapshot NAME Snapshot to restore.");
|
||||
System.err.println(" -copy-to NAME Remote destination hdfs://");
|
||||
System.err.println(" -copy-from NAME Input folder hdfs:// (default hbase.rootdir)");
|
||||
System.err.println(" -no-checksum-verify Do not verify checksum, use name+length only.");
|
||||
System.err.println(" -no-target-verify Do not verify the integrity of the \\" +
|
||||
"exported snapshot.");
|
||||
System.err.println(" -overwrite Rewrite the snapshot manifest if already exists");
|
||||
System.err.println(" -chuser USERNAME Change the owner of the files " +
|
||||
"to the specified one.");
|
||||
System.err.println(" -chgroup GROUP Change the group of the files to " +
|
||||
"the specified one.");
|
||||
System.err.println(" -chmod MODE Change the permission of the files " +
|
||||
"to the specified one.");
|
||||
System.err.println(" -mappers Number of mappers to use during the " +
|
||||
"copy (mapreduce.job.maps).");
|
||||
System.err.println(" -bandwidth Limit bandwidth to this value in MB/second.");
|
||||
System.err.println();
|
||||
System.err.println("Examples:");
|
||||
System.err.println(" hbase snapshot export \\");
|
||||
System.err.println(" -snapshot MySnapshot -copy-to hdfs://srv2:8082/hbase \\");
|
||||
System.err.println(" -chuser MyUser -chgroup MyGroup -chmod 700 -mappers 16");
|
||||
System.err.println();
|
||||
System.err.println(" hbase snapshot export \\");
|
||||
System.err.println(" -snapshot MySnapshot -copy-from hdfs://srv2:8082/hbase \\");
|
||||
System.err.println(" -copy-to hdfs://srv1:50070/hbase \\");
|
||||
@Override
|
||||
protected void printUsage() {
|
||||
super.printUsage();
|
||||
System.out.println("\n"
|
||||
+ "Examples:\n"
|
||||
+ " hbase snapshot export \\\n"
|
||||
+ " --snapshot MySnapshot --copy-to hdfs://srv2:8082/hbase \\\n"
|
||||
+ " --chuser MyUser --chgroup MyGroup --chmod 700 --mappers 16\n"
|
||||
+ "\n"
|
||||
+ " hbase snapshot export \\\n"
|
||||
+ " --snapshot MySnapshot --copy-from hdfs://srv2:8082/hbase \\\n"
|
||||
+ " --copy-to hdfs://srv1:50070/hbase");
|
||||
System.exit(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* The guts of the {@link #main} method.
|
||||
* Call this method to avoid the {@link #main(String[])} System.exit.
|
||||
* @param args
|
||||
* @return errCode
|
||||
* @throws Exception
|
||||
*/
|
||||
static int innerMain(final Configuration conf, final String [] args) throws Exception {
|
||||
return ToolRunner.run(conf, new ExportSnapshot(), args);
|
||||
@Override protected void addOptions() {
|
||||
addRequiredOption(Options.SNAPSHOT);
|
||||
addOption(Options.COPY_TO);
|
||||
addOption(Options.COPY_FROM);
|
||||
addOption(Options.TARGET_NAME);
|
||||
addOption(Options.NO_CHECKSUM_VERIFY);
|
||||
addOption(Options.NO_TARGET_VERIFY);
|
||||
addOption(Options.OVERWRITE);
|
||||
addOption(Options.CHUSER);
|
||||
addOption(Options.CHGROUP);
|
||||
addOption(Options.CHMOD);
|
||||
addOption(Options.MAPPERS);
|
||||
addOption(Options.BANDWIDTH);
|
||||
}
|
||||
|
||||
public static void main(String[] args) throws Exception {
|
||||
System.exit(innerMain(HBaseConfiguration.create(), args));
|
||||
public static void main(String[] args) {
|
||||
new ExportSnapshot().doStaticMain(args);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
package org.apache.hadoop.hbase.snapshot;
|
||||
|
||||
import static org.apache.hadoop.util.ToolRunner.run;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
|
|
@ -48,6 +49,7 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.SnapshotProtos.Snapshot
|
|||
import org.apache.hadoop.hbase.testclassification.VerySlowMapReduceTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.FSUtils;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Before;
|
||||
|
|
@ -204,18 +206,18 @@ public class TestExportSnapshot {
|
|||
copyDir = copyDir.makeQualified(fs);
|
||||
|
||||
List<String> opts = new ArrayList<String>();
|
||||
opts.add("-snapshot");
|
||||
opts.add("--snapshot");
|
||||
opts.add(Bytes.toString(snapshotName));
|
||||
opts.add("-copy-to");
|
||||
opts.add("--copy-to");
|
||||
opts.add(copyDir.toString());
|
||||
if (targetName != snapshotName) {
|
||||
opts.add("-target");
|
||||
opts.add("--target");
|
||||
opts.add(Bytes.toString(targetName));
|
||||
}
|
||||
if (overwrite) opts.add("-overwrite");
|
||||
if (overwrite) opts.add("--overwrite");
|
||||
|
||||
// Export Snapshot
|
||||
int res = ExportSnapshot.innerMain(conf, opts.toArray(new String[opts.size()]));
|
||||
int res = run(conf, new ExportSnapshot(), opts.toArray(new String[opts.size()]));
|
||||
assertEquals(0, res);
|
||||
|
||||
// Verify File-System state
|
||||
|
|
@ -273,12 +275,9 @@ public class TestExportSnapshot {
|
|||
}
|
||||
// Export Snapshot
|
||||
Path sourceDir = TEST_UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
|
||||
int res = ExportSnapshot.innerMain(conf, new String[] {
|
||||
"-snapshot", Bytes.toString(snapshotName),
|
||||
"-copy-from", sourceDir.toString(),
|
||||
"-copy-to", copyDir.toString()
|
||||
});
|
||||
return res;
|
||||
String[] args = new String[] { "--snapshot", Bytes.toString(snapshotName),
|
||||
"--copy-from", sourceDir.toString(), "--copy-to", copyDir.toString() };
|
||||
return ToolRunner.run(conf, new ExportSnapshot(), args);
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
|
|||
Loading…
Reference in New Issue