Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16772 Add verbose option to VerifyReplication for logging good rows

This commit is contained in:
tedyu 2016-10-06 11:16:18 -07:00
parent 7a38509782
commit 97c1333831
1 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/replication/VerifyReplication.java
View File

@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@ -47,8 +46,8 @@ import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
@ -86,6 +85,7 @@ public class VerifyReplication extends Configured implements Tool {
static String peerId = null;
static String rowPrefixes = null;
static int sleepMsBeforeReCompare = 0;
static boolean verbose = false;
private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name";
@ -107,6 +107,7 @@ public class VerifyReplication extends Configured implements Tool {
private ResultScanner replicatedScanner;
private Result currentCompareRowInPeerTable;
private int sleepMsBeforeReCompare;
private boolean verbose = false;
/**
* Map method that compares every scanned row with the equivalent from
@ -123,6 +124,7 @@ public class VerifyReplication extends Configured implements Tool {
if (replicatedScanner == null) {
Configuration conf = context.getConfiguration();
sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0);
verbose = conf.getBoolean(NAME +".verbose", false);
final Scan scan = new Scan();
scan.setBatch(batch);
scan.setCacheBlocks(false);
@ -173,6 +175,9 @@ public class VerifyReplication extends Configured implements Tool {
try {
Result.compareResults(value, currentCompareRowInPeerTable);
context.getCounter(Counters.GOODROWS).increment(1);
if (verbose) {
LOG.info("Good row key: " + delimiter + Bytes.toString(value.getRow()) + delimiter);
}
} catch (Exception e) {
logFailRowAndIncreaseCounter(context, Counters.CONTENT_DIFFERENT_ROWS, value);
LOG.error("Exception while comparing row : " + e);
@ -199,6 +204,10 @@ public class VerifyReplication extends Configured implements Tool {
Result sourceResult = sourceTable.get(new Get(row.getRow()));
Result replicatedResult = replicatedTable.get(new Get(row.getRow()));
Result.compareResults(sourceResult, replicatedResult);
context.getCounter(Counters.GOODROWS).increment(1);
if (verbose) {
LOG.info("Good row key: " + delimiter + Bytes.toString(row.getRow()) + delimiter);
}
return;
} catch (Exception e) {
LOG.error("recompare fail after sleep, rowkey=" + delimiter +
@ -311,6 +320,7 @@ public class VerifyReplication extends Configured implements Tool {
conf.setLong(NAME+".startTime", startTime);
conf.setLong(NAME+".endTime", endTime);
conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare);
conf.setBoolean(NAME +".verbose", verbose);
if (families != null) {
conf.set(NAME+".families", families);
}
@ -451,7 +461,11 @@ public class VerifyReplication extends Configured implements Tool {
sleepMsBeforeReCompare = Integer.parseInt(cmd.substring(sleepToReCompareKey.length()));
continue;
}
final String verboseKey = "--verbose";
if (cmd.startsWith(verboseKey)) {
verbose = true;
continue;
}
if (i == args.length-2) {
peerId = cmd;
}
@ -487,7 +501,8 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println("ERROR: " + errorMsg);
}
System.err.println("Usage: verifyrep [--starttime=X]" +
" [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] <peerid> <tablename>");
" [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] [--recomparesleep=] " +
"[--verbose] <peerid> <tablename>");
System.err.println();
System.err.println("Options:");
System.err.println(" starttime beginning of the time range");
@ -499,6 +514,7 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println(" delimiter the delimiter used in display around rowkey");
System.err.println(" recomparesleep milliseconds to sleep before recompare row, " +
"default value is 0 which disables the recompare.");
System.err.println(" verbose logs row keys of good rows");
System.err.println();
System.err.println("Args:");
System.err.println(" peerid Id of the peer used for verification, must match the one given for replication");