HBASE-16772 Add verbose option to VerifyReplication for logging good rows

This commit is contained in:
tedyu 2016-10-06 11:16:18 -07:00
parent 7a38509782
commit 97c1333831
1 changed files with 20 additions and 4 deletions

View File

@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured; import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.hbase.Abortable; import org.apache.hadoop.hbase.Abortable;
import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.ConnectionFactory;
@ -47,8 +46,8 @@ import org.apache.hadoop.hbase.mapreduce.TableMapper;
import org.apache.hadoop.hbase.mapreduce.TableSplit; import org.apache.hadoop.hbase.mapreduce.TableSplit;
import org.apache.hadoop.hbase.replication.ReplicationException; import org.apache.hadoop.hbase.replication.ReplicationException;
import org.apache.hadoop.hbase.replication.ReplicationFactory; import org.apache.hadoop.hbase.replication.ReplicationFactory;
import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig; import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
import org.apache.hadoop.hbase.replication.ReplicationPeers; import org.apache.hadoop.hbase.replication.ReplicationPeers;
import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair; import org.apache.hadoop.hbase.util.Pair;
@ -86,6 +85,7 @@ public class VerifyReplication extends Configured implements Tool {
static String peerId = null; static String peerId = null;
static String rowPrefixes = null; static String rowPrefixes = null;
static int sleepMsBeforeReCompare = 0; static int sleepMsBeforeReCompare = 0;
static boolean verbose = false;
private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name"; private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name";
@ -107,6 +107,7 @@ public class VerifyReplication extends Configured implements Tool {
private ResultScanner replicatedScanner; private ResultScanner replicatedScanner;
private Result currentCompareRowInPeerTable; private Result currentCompareRowInPeerTable;
private int sleepMsBeforeReCompare; private int sleepMsBeforeReCompare;
private boolean verbose = false;
/** /**
* Map method that compares every scanned row with the equivalent from * Map method that compares every scanned row with the equivalent from
@ -123,6 +124,7 @@ public class VerifyReplication extends Configured implements Tool {
if (replicatedScanner == null) { if (replicatedScanner == null) {
Configuration conf = context.getConfiguration(); Configuration conf = context.getConfiguration();
sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0); sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0);
verbose = conf.getBoolean(NAME +".verbose", false);
final Scan scan = new Scan(); final Scan scan = new Scan();
scan.setBatch(batch); scan.setBatch(batch);
scan.setCacheBlocks(false); scan.setCacheBlocks(false);
@ -173,6 +175,9 @@ public class VerifyReplication extends Configured implements Tool {
try { try {
Result.compareResults(value, currentCompareRowInPeerTable); Result.compareResults(value, currentCompareRowInPeerTable);
context.getCounter(Counters.GOODROWS).increment(1); context.getCounter(Counters.GOODROWS).increment(1);
if (verbose) {
LOG.info("Good row key: " + delimiter + Bytes.toString(value.getRow()) + delimiter);
}
} catch (Exception e) { } catch (Exception e) {
logFailRowAndIncreaseCounter(context, Counters.CONTENT_DIFFERENT_ROWS, value); logFailRowAndIncreaseCounter(context, Counters.CONTENT_DIFFERENT_ROWS, value);
LOG.error("Exception while comparing row : " + e); LOG.error("Exception while comparing row : " + e);
@ -199,6 +204,10 @@ public class VerifyReplication extends Configured implements Tool {
Result sourceResult = sourceTable.get(new Get(row.getRow())); Result sourceResult = sourceTable.get(new Get(row.getRow()));
Result replicatedResult = replicatedTable.get(new Get(row.getRow())); Result replicatedResult = replicatedTable.get(new Get(row.getRow()));
Result.compareResults(sourceResult, replicatedResult); Result.compareResults(sourceResult, replicatedResult);
context.getCounter(Counters.GOODROWS).increment(1);
if (verbose) {
LOG.info("Good row key: " + delimiter + Bytes.toString(row.getRow()) + delimiter);
}
return; return;
} catch (Exception e) { } catch (Exception e) {
LOG.error("recompare fail after sleep, rowkey=" + delimiter + LOG.error("recompare fail after sleep, rowkey=" + delimiter +
@ -311,6 +320,7 @@ public class VerifyReplication extends Configured implements Tool {
conf.setLong(NAME+".startTime", startTime); conf.setLong(NAME+".startTime", startTime);
conf.setLong(NAME+".endTime", endTime); conf.setLong(NAME+".endTime", endTime);
conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare); conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare);
conf.setBoolean(NAME +".verbose", verbose);
if (families != null) { if (families != null) {
conf.set(NAME+".families", families); conf.set(NAME+".families", families);
} }
@ -451,7 +461,11 @@ public class VerifyReplication extends Configured implements Tool {
sleepMsBeforeReCompare = Integer.parseInt(cmd.substring(sleepToReCompareKey.length())); sleepMsBeforeReCompare = Integer.parseInt(cmd.substring(sleepToReCompareKey.length()));
continue; continue;
} }
final String verboseKey = "--verbose";
if (cmd.startsWith(verboseKey)) {
verbose = true;
continue;
}
if (i == args.length-2) { if (i == args.length-2) {
peerId = cmd; peerId = cmd;
} }
@ -487,7 +501,8 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println("ERROR: " + errorMsg); System.err.println("ERROR: " + errorMsg);
} }
System.err.println("Usage: verifyrep [--starttime=X]" + System.err.println("Usage: verifyrep [--starttime=X]" +
" [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] <peerid> <tablename>"); " [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] [--recomparesleep=] " +
"[--verbose] <peerid> <tablename>");
System.err.println(); System.err.println();
System.err.println("Options:"); System.err.println("Options:");
System.err.println(" starttime beginning of the time range"); System.err.println(" starttime beginning of the time range");
@ -499,6 +514,7 @@ public class VerifyReplication extends Configured implements Tool {
System.err.println(" delimiter the delimiter used in display around rowkey"); System.err.println(" delimiter the delimiter used in display around rowkey");
System.err.println(" recomparesleep milliseconds to sleep before recompare row, " + System.err.println(" recomparesleep milliseconds to sleep before recompare row, " +
"default value is 0 which disables the recompare."); "default value is 0 which disables the recompare.");
System.err.println(" verbose logs row keys of good rows");
System.err.println(); System.err.println();
System.err.println("Args:"); System.err.println("Args:");
System.err.println(" peerid Id of the peer used for verification, must match the one given for replication"); System.err.println(" peerid Id of the peer used for verification, must match the one given for replication");