HBASE-16772 Add verbose option to VerifyReplication for logging good rows
This commit is contained in:
parent
7a38509782
commit
97c1333831
|
@ -27,7 +27,6 @@ import org.apache.hadoop.conf.Configuration;
|
||||||
import org.apache.hadoop.conf.Configured;
|
import org.apache.hadoop.conf.Configured;
|
||||||
import org.apache.hadoop.hbase.Abortable;
|
import org.apache.hadoop.hbase.Abortable;
|
||||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||||
import org.apache.hadoop.hbase.HConstants;
|
|
||||||
import org.apache.hadoop.hbase.TableName;
|
import org.apache.hadoop.hbase.TableName;
|
||||||
import org.apache.hadoop.hbase.client.Connection;
|
import org.apache.hadoop.hbase.client.Connection;
|
||||||
import org.apache.hadoop.hbase.client.ConnectionFactory;
|
import org.apache.hadoop.hbase.client.ConnectionFactory;
|
||||||
|
@ -47,8 +46,8 @@ import org.apache.hadoop.hbase.mapreduce.TableMapper;
|
||||||
import org.apache.hadoop.hbase.mapreduce.TableSplit;
|
import org.apache.hadoop.hbase.mapreduce.TableSplit;
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationException;
|
import org.apache.hadoop.hbase.replication.ReplicationException;
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationFactory;
|
import org.apache.hadoop.hbase.replication.ReplicationFactory;
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
|
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
|
import org.apache.hadoop.hbase.replication.ReplicationPeerConfig;
|
||||||
|
import org.apache.hadoop.hbase.replication.ReplicationPeerZKImpl;
|
||||||
import org.apache.hadoop.hbase.replication.ReplicationPeers;
|
import org.apache.hadoop.hbase.replication.ReplicationPeers;
|
||||||
import org.apache.hadoop.hbase.util.Bytes;
|
import org.apache.hadoop.hbase.util.Bytes;
|
||||||
import org.apache.hadoop.hbase.util.Pair;
|
import org.apache.hadoop.hbase.util.Pair;
|
||||||
|
@ -86,6 +85,7 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
static String peerId = null;
|
static String peerId = null;
|
||||||
static String rowPrefixes = null;
|
static String rowPrefixes = null;
|
||||||
static int sleepMsBeforeReCompare = 0;
|
static int sleepMsBeforeReCompare = 0;
|
||||||
|
static boolean verbose = false;
|
||||||
|
|
||||||
private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name";
|
private final static String JOB_NAME_CONF_KEY = "mapreduce.job.name";
|
||||||
|
|
||||||
|
@ -107,6 +107,7 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
private ResultScanner replicatedScanner;
|
private ResultScanner replicatedScanner;
|
||||||
private Result currentCompareRowInPeerTable;
|
private Result currentCompareRowInPeerTable;
|
||||||
private int sleepMsBeforeReCompare;
|
private int sleepMsBeforeReCompare;
|
||||||
|
private boolean verbose = false;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Map method that compares every scanned row with the equivalent from
|
* Map method that compares every scanned row with the equivalent from
|
||||||
|
@ -123,6 +124,7 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
if (replicatedScanner == null) {
|
if (replicatedScanner == null) {
|
||||||
Configuration conf = context.getConfiguration();
|
Configuration conf = context.getConfiguration();
|
||||||
sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0);
|
sleepMsBeforeReCompare = conf.getInt(NAME +".sleepMsBeforeReCompare", 0);
|
||||||
|
verbose = conf.getBoolean(NAME +".verbose", false);
|
||||||
final Scan scan = new Scan();
|
final Scan scan = new Scan();
|
||||||
scan.setBatch(batch);
|
scan.setBatch(batch);
|
||||||
scan.setCacheBlocks(false);
|
scan.setCacheBlocks(false);
|
||||||
|
@ -173,6 +175,9 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
try {
|
try {
|
||||||
Result.compareResults(value, currentCompareRowInPeerTable);
|
Result.compareResults(value, currentCompareRowInPeerTable);
|
||||||
context.getCounter(Counters.GOODROWS).increment(1);
|
context.getCounter(Counters.GOODROWS).increment(1);
|
||||||
|
if (verbose) {
|
||||||
|
LOG.info("Good row key: " + delimiter + Bytes.toString(value.getRow()) + delimiter);
|
||||||
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logFailRowAndIncreaseCounter(context, Counters.CONTENT_DIFFERENT_ROWS, value);
|
logFailRowAndIncreaseCounter(context, Counters.CONTENT_DIFFERENT_ROWS, value);
|
||||||
LOG.error("Exception while comparing row : " + e);
|
LOG.error("Exception while comparing row : " + e);
|
||||||
|
@ -199,6 +204,10 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
Result sourceResult = sourceTable.get(new Get(row.getRow()));
|
Result sourceResult = sourceTable.get(new Get(row.getRow()));
|
||||||
Result replicatedResult = replicatedTable.get(new Get(row.getRow()));
|
Result replicatedResult = replicatedTable.get(new Get(row.getRow()));
|
||||||
Result.compareResults(sourceResult, replicatedResult);
|
Result.compareResults(sourceResult, replicatedResult);
|
||||||
|
context.getCounter(Counters.GOODROWS).increment(1);
|
||||||
|
if (verbose) {
|
||||||
|
LOG.info("Good row key: " + delimiter + Bytes.toString(row.getRow()) + delimiter);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
LOG.error("recompare fail after sleep, rowkey=" + delimiter +
|
LOG.error("recompare fail after sleep, rowkey=" + delimiter +
|
||||||
|
@ -311,6 +320,7 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
conf.setLong(NAME+".startTime", startTime);
|
conf.setLong(NAME+".startTime", startTime);
|
||||||
conf.setLong(NAME+".endTime", endTime);
|
conf.setLong(NAME+".endTime", endTime);
|
||||||
conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare);
|
conf.setInt(NAME +".sleepMsBeforeReCompare", sleepMsBeforeReCompare);
|
||||||
|
conf.setBoolean(NAME +".verbose", verbose);
|
||||||
if (families != null) {
|
if (families != null) {
|
||||||
conf.set(NAME+".families", families);
|
conf.set(NAME+".families", families);
|
||||||
}
|
}
|
||||||
|
@ -451,7 +461,11 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
sleepMsBeforeReCompare = Integer.parseInt(cmd.substring(sleepToReCompareKey.length()));
|
sleepMsBeforeReCompare = Integer.parseInt(cmd.substring(sleepToReCompareKey.length()));
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
final String verboseKey = "--verbose";
|
||||||
|
if (cmd.startsWith(verboseKey)) {
|
||||||
|
verbose = true;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if (i == args.length-2) {
|
if (i == args.length-2) {
|
||||||
peerId = cmd;
|
peerId = cmd;
|
||||||
}
|
}
|
||||||
|
@ -487,7 +501,8 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
System.err.println("ERROR: " + errorMsg);
|
System.err.println("ERROR: " + errorMsg);
|
||||||
}
|
}
|
||||||
System.err.println("Usage: verifyrep [--starttime=X]" +
|
System.err.println("Usage: verifyrep [--starttime=X]" +
|
||||||
" [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] <peerid> <tablename>");
|
" [--endtime=Y] [--families=A] [--row-prefixes=B] [--delimiter=] [--recomparesleep=] " +
|
||||||
|
"[--verbose] <peerid> <tablename>");
|
||||||
System.err.println();
|
System.err.println();
|
||||||
System.err.println("Options:");
|
System.err.println("Options:");
|
||||||
System.err.println(" starttime beginning of the time range");
|
System.err.println(" starttime beginning of the time range");
|
||||||
|
@ -499,6 +514,7 @@ public class VerifyReplication extends Configured implements Tool {
|
||||||
System.err.println(" delimiter the delimiter used in display around rowkey");
|
System.err.println(" delimiter the delimiter used in display around rowkey");
|
||||||
System.err.println(" recomparesleep milliseconds to sleep before recompare row, " +
|
System.err.println(" recomparesleep milliseconds to sleep before recompare row, " +
|
||||||
"default value is 0 which disables the recompare.");
|
"default value is 0 which disables the recompare.");
|
||||||
|
System.err.println(" verbose logs row keys of good rows");
|
||||||
System.err.println();
|
System.err.println();
|
||||||
System.err.println("Args:");
|
System.err.println("Args:");
|
||||||
System.err.println(" peerid Id of the peer used for verification, must match the one given for replication");
|
System.err.println(" peerid Id of the peer used for verification, must match the one given for replication");
|
||||||
|
|
Loading…
Reference in New Issue