HBASE-5886 Add new metric for possible data loss due to puts without WAL

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1333676 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael Stack 2012-05-03 23:01:10 +00:00
parent 05433f891b
commit d45f53a307
3 changed files with 60 additions and 3 deletions

View File

@ -220,6 +220,10 @@ public class HRegion implements HeapSize { // , Writable{
final AtomicLong memstoreSize = new AtomicLong(0);
// Debug possible data loss due to WAL off
final AtomicLong numPutsWithoutWAL = new AtomicLong(0);
final AtomicLong dataInMemoryWithoutWAL = new AtomicLong(0);
final Counter readRequestsCount = new Counter();
final Counter writeRequestsCount = new Counter();
@ -1316,6 +1320,10 @@ public class HRegion implements HeapSize { // , Writable{
status.setStatus("Running coprocessor pre-flush hooks");
coprocessorHost.preFlush();
}
if (numPutsWithoutWAL.get() > 0) {
numPutsWithoutWAL.set(0);
dataInMemoryWithoutWAL.set(0);
}
synchronized (writestate) {
if (!writestate.flushing && writestate.writesEnabled) {
this.writestate.flushing = true;
@ -2169,7 +2177,10 @@ public class HRegion implements HeapSize { // , Writable{
batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
Put p = batchOp.operations[i].getFirst();
if (!p.getWriteToWAL()) continue;
if (!p.getWriteToWAL()) {
recordPutWithoutWal(p.getFamilyMap());
continue;
}
// Add WAL edits by CP
WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];
if (fromCP != null) {
@ -2502,6 +2513,8 @@ public class HRegion implements HeapSize { // , Writable{
addFamilyMapToWALEdit(familyMap, walEdit);
this.log.append(regionInfo, this.htableDescriptor.getName(),
walEdit, clusterId, now, this.htableDescriptor);
} else {
recordPutWithoutWal(familyMap);
}
long addedSize = applyFamilyMapToMemstore(familyMap, null);
@ -4830,14 +4843,14 @@ public class HRegion implements HeapSize { // , Writable{
public static final long FIXED_OVERHEAD = ClassSize.align(
ClassSize.OBJECT +
ClassSize.ARRAY +
34 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
36 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
(6 * Bytes.SIZEOF_LONG) +
Bytes.SIZEOF_BOOLEAN);
public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
ClassSize.OBJECT + // closeLock
(2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing
ClassSize.ATOMIC_LONG + // memStoreSize
(3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL
ClassSize.ATOMIC_INTEGER + // lockIdGenerator
(3 * ClassSize.CONCURRENT_HASHMAP) + // lockedRows, lockIds, scannerReadPoints
WriteState.HEAP_SIZE + // writestate
@ -5185,6 +5198,26 @@ public class HRegion implements HeapSize { // , Writable{
else lock.readLock().unlock();
}
/**
* Update counters for numer of puts without wal and the size of possible data loss.
* These information are exposed by the region server metrics.
*/
private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
if (numPutsWithoutWAL.getAndIncrement() == 0) {
LOG.info("writing data to region " + this +
" with WAL disabled. Data may be lost in the event of a crash.");
}
long putSize = 0;
for (List<KeyValue> edits : familyMap.values()) {
for (KeyValue kv : edits) {
putSize += kv.getKeyLength() + kv.getValueLength();
}
}
dataInMemoryWithoutWAL.addAndGet(putSize);
}
/**
* A mocked list implementaion - discards all updates.
*/

View File

@ -1187,6 +1187,8 @@ public class HRegionServer extends RegionServer
new HDFSBlocksDistribution();
long totalStaticIndexSize = 0;
long totalStaticBloomSize = 0;
long numPutsWithoutWAL = 0;
long dataInMemoryWithoutWAL = 0;
// Note that this is a map of Doubles instead of Longs. This is because we
// do effective integer division, which would perhaps truncate more than it
@ -1199,6 +1201,8 @@ public class HRegionServer extends RegionServer
for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
HRegion r = e.getValue();
memstoreSize += r.memstoreSize.get();
numPutsWithoutWAL += r.numPutsWithoutWAL.get();
dataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
readRequestsCount += r.readRequestsCount.get();
writeRequestsCount += r.writeRequestsCount.get();
synchronized (r.stores) {
@ -1262,6 +1266,8 @@ public class HRegionServer extends RegionServer
this.metrics.stores.set(stores);
this.metrics.storefiles.set(storefiles);
this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
this.metrics.mbInMemoryWithoutWAL.set((int) (dataInMemoryWithoutWAL / (1024 * 1024)));
this.metrics.numPutsWithoutWAL.set(numPutsWithoutWAL);
this.metrics.storefileIndexSizeMB.set(
(int) (storefileIndexSize / (1024 * 1024)));
this.metrics.rootIndexSizeKB.set(

View File

@ -209,6 +209,18 @@ public class RegionServerMetrics implements Updater {
public final MetricsIntValue memstoreSizeMB =
new MetricsIntValue("memstoreSizeMB", registry);
/**
* Number of put with WAL disabled in this regionserver in MB
*/
public final MetricsLongValue numPutsWithoutWAL =
new MetricsLongValue("numPutsWithoutWAL", registry);
/**
* Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB
*/
public final MetricsIntValue mbInMemoryWithoutWAL =
new MetricsIntValue("mbInMemoryWithoutWAL", registry);
/**
* Size of the compaction queue.
*/
@ -366,6 +378,8 @@ public class RegionServerMetrics implements Updater {
this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord);
this.memstoreSizeMB.pushMetric(this.metricsRecord);
this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord);
this.numPutsWithoutWAL.pushMetric(this.metricsRecord);
this.readRequestsCount.pushMetric(this.metricsRecord);
this.writeRequestsCount.pushMetric(this.metricsRecord);
this.regions.pushMetric(this.metricsRecord);
@ -535,6 +549,10 @@ public class RegionServerMetrics implements Updater {
Integer.valueOf(this.totalStaticBloomSizeKB.get()));
sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(),
Integer.valueOf(this.memstoreSizeMB.get()));
sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL",
Integer.valueOf(this.mbInMemoryWithoutWAL.get()));
sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL",
Long.valueOf(this.numPutsWithoutWAL.get()));
sb = Strings.appendKeyValue(sb, "readRequestsCount",
Long.valueOf(this.readRequestsCount.get()));
sb = Strings.appendKeyValue(sb, "writeRequestsCount",