Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-17826 Backup: submit M/R job to a particular Yarn queue (Vladimir Rodionov)

This commit is contained in:
tedyu 2017-08-30 16:11:32 -07:00
parent b5942241ca
commit 645e5a5e2b
4 changed files with 39 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
View File

@ -67,6 +67,11 @@ public final class BackupRequest {
return this;
}
public Builder withYarnPoolName(String name) {
request.setYarnPoolName(name);
return this;
}
public BackupRequest build() {
return request;
}
@ -79,6 +84,7 @@ public final class BackupRequest {
private int totalTasks = -1;
private long bandwidth = -1L;
private String backupSetName;
private String yarnPoolName;
private BackupRequest() {
}
@ -136,4 +142,12 @@ public final class BackupRequest {
this.backupSetName = backupSetName;
return this;
}
public String getYarnPoolName() {
return yarnPoolName;
}
public void setYarnPoolName(String yarnPoolName) {
this.yarnPoolName = yarnPoolName;
}
}

4
hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
View File

@ -89,6 +89,10 @@ public interface BackupRestoreConstants {
public static final String OPTION_TABLE_MAPPING_DESC =
"A comma separated list of target tables. "
+ "If specified, each table in <tables> must have a mapping";
public static final String OPTION_YARN_QUEUE_NAME = "q";
public static final String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup create command on";
public static final String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run backup restore command on";
public static final String BACKUP_CONFIG_STRING = BackupRestoreConstants.BACKUP_ENABLE_KEY + "=true\n"
+ "hbase.master.logcleaner.plugins="

12
hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
View File

@ -29,6 +29,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_RESTORE_DESC;
import java.io.IOException;
import java.net.URI;
@ -125,6 +127,13 @@ public class RestoreDriver extends AbstractHBaseTool {
printToolUsage();
return -1;
}
if (cmd.hasOption(OPTION_YARN_QUEUE_NAME)) {
String queueName = cmd.getOptionValue(OPTION_YARN_QUEUE_NAME);
// Set system property value for MR job
System.setProperty("mapreduce.job.queuename", queueName);
}
// parse main restore command options
String[] remainArgs = cmd.getArgs();
if (remainArgs.length != 2) {
@ -195,8 +204,9 @@ public class RestoreDriver extends AbstractHBaseTool {
addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC);
addOptWithArg(OPTION_SET, OPTION_SET_RESTORE_DESC);
addOptWithArg(OPTION_TABLE, OPTION_TABLE_LIST_DESC);
addOptWithArg(OPTION_TABLE_MAPPING, OPTION_TABLE_MAPPING_DESC);
addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_RESTORE_DESC);
}
@Override

10
hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
View File

@ -32,6 +32,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
import java.io.IOException;
import java.net.URI;
@ -321,6 +323,12 @@ public final class BackupCommands {
cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline
.getOptionValue(OPTION_WORKERS)) : -1;
if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
String poolName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
// Set system property value for MR job
System.setProperty("mapreduce.job.queuename", poolName);
}
try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
BackupRequest.Builder builder = new BackupRequest.Builder();
@ -368,6 +376,8 @@ public final class BackupCommands {
options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setLeftPadding(2);