Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-15191 CopyTable and VerifyReplication - Option to specify batch size, versions (Parth Shah)

This commit is contained in:
tedyu 2016-03-28 17:25:00 -07:00
parent 7793bc54c2
commit cbf9c1e116
2 changed files with 35 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/CopyTable.java
View File

@ -54,7 +54,9 @@ public class CopyTable extends Configured implements Tool {
final static String NAME = "copytable"; final static String NAME = "copytable";
long startTime = 0; long startTime = 0;
long endTime = 0; long endTime = HConstants.LATEST_TIMESTAMP;
int batch = Integer.MAX_VALUE;
int cacheRow = -1;
int versions = -1; int versions = -1;
String tableName = null; String tableName = null;
String startRow = null; String startRow = null;
@ -82,15 +84,22 @@ public class CopyTable extends Configured implements Tool {
if (!doCommandLine(args)) { if (!doCommandLine(args)) {
return null; return null;
} }
Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName)); Job job = Job.getInstance(getConf(), getConf().get(JOB_NAME_CONF_KEY, NAME + "_" + tableName));
job.setJarByClass(CopyTable.class); job.setJarByClass(CopyTable.class);
Scan scan = new Scan(); Scan scan = new Scan();
scan.setBatch(batch);
scan.setCacheBlocks(false); scan.setCacheBlocks(false);
if (startTime != 0) {
scan.setTimeRange(startTime, if (cacheRow > 0) {
endTime == 0 ? HConstants.LATEST_TIMESTAMP : endTime); scan.setCaching(cacheRow);
} else {
scan.setCaching(getConf().getInt(HConstants.HBASE_CLIENT_SCANNER_CACHING, 100));
} }
scan.setTimeRange(startTime, endTime);
if (allCells) { if (allCells) {
scan.setRaw(true); scan.setRaw(true);
} }
@ -252,6 +261,18 @@ public class CopyTable extends Configured implements Tool {
endTime = Long.parseLong(cmd.substring(endTimeArgKey.length())); endTime = Long.parseLong(cmd.substring(endTimeArgKey.length()));
continue; continue;
} }
final String batchArgKey = "--batch=";
if (cmd.startsWith(batchArgKey)) {
batch = Integer.parseInt(cmd.substring(batchArgKey.length()));
continue;
}
final String cacheRowArgKey = "--cacheRow=";
if (cmd.startsWith(cacheRowArgKey)) {
cacheRow = Integer.parseInt(cmd.substring(cacheRowArgKey.length()));
continue;
}
final String versionsArgKey = "--versions="; final String versionsArgKey = "--versions=";
if (cmd.startsWith(versionsArgKey)) { if (cmd.startsWith(versionsArgKey)) {

9
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
View File

@ -72,6 +72,7 @@ public class VerifyReplication extends Configured implements Tool {
private final static String PEER_CONFIG_PREFIX = NAME + ".peer."; private final static String PEER_CONFIG_PREFIX = NAME + ".peer.";
static long startTime = 0; static long startTime = 0;
static long endTime = Long.MAX_VALUE; static long endTime = Long.MAX_VALUE;
static int batch = Integer.MAX_VALUE;
static int versions = -1; static int versions = -1;
static String tableName = null; static String tableName = null;
static String families = null; static String families = null;
@ -110,6 +111,8 @@ public class VerifyReplication extends Configured implements Tool {
if (replicatedScanner == null) { if (replicatedScanner == null) {
Configuration conf = context.getConfiguration(); Configuration conf = context.getConfiguration();
final Scan scan = new Scan(); final Scan scan = new Scan();
scan.setBatch(batch);
scan.setCacheBlocks(false);
scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1)); scan.setCaching(conf.getInt(TableInputFormat.SCAN_CACHEDROWS, 1));
long startTime = conf.getLong(NAME + ".startTime", 0); long startTime = conf.getLong(NAME + ".startTime", 0);
long endTime = conf.getLong(NAME + ".endTime", Long.MAX_VALUE); long endTime = conf.getLong(NAME + ".endTime", Long.MAX_VALUE);
@ -338,6 +341,12 @@ public class VerifyReplication extends Configured implements Tool {
versions = Integer.parseInt(cmd.substring(versionsArgKey.length())); versions = Integer.parseInt(cmd.substring(versionsArgKey.length()));
continue; continue;
} }
final String batchArgKey = "--batch=";
if (cmd.startsWith(batchArgKey)) {
batch = Integer.parseInt(cmd.substring(batchArgKey.length()));
continue;
}
final String familiesArgKey = "--families="; final String familiesArgKey = "--families=";
if (cmd.startsWith(familiesArgKey)) { if (cmd.startsWith(familiesArgKey)) {