HBASE-5886 Add new metric for possible data loss due to puts without WAL

git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1333676 13f79535-47bb-0310-9956-ffa450edef68

src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java

@@ -220,6 +220,10 @@ public class HRegion implements HeapSize { // , Writable{
 
   final AtomicLong memstoreSize = new AtomicLong(0);
 
+  // Debug possible data loss due to WAL off
+  final AtomicLong numPutsWithoutWAL = new AtomicLong(0);
+  final AtomicLong dataInMemoryWithoutWAL = new AtomicLong(0);
+
   final Counter readRequestsCount = new Counter();
   final Counter writeRequestsCount = new Counter();
 
@@ -1316,6 +1320,10 @@ public class HRegion implements HeapSize { // , Writable{
         status.setStatus("Running coprocessor pre-flush hooks");
         coprocessorHost.preFlush();
       }
+      if (numPutsWithoutWAL.get() > 0) {
+        numPutsWithoutWAL.set(0);
+        dataInMemoryWithoutWAL.set(0);
+      }
       synchronized (writestate) {
         if (!writestate.flushing && writestate.writesEnabled) {
           this.writestate.flushing = true;
@@ -2169,7 +2177,10 @@ public class HRegion implements HeapSize { // , Writable{
         batchOp.retCodeDetails[i] = OperationStatus.SUCCESS;
 
         Put p = batchOp.operations[i].getFirst();
-        if (!p.getWriteToWAL()) continue;
+        if (!p.getWriteToWAL()) {
+          recordPutWithoutWal(p.getFamilyMap());
+          continue;
+        }
         // Add WAL edits by CP
         WALEdit fromCP = batchOp.walEditsFromCoprocessors[i];
         if (fromCP != null) {
@@ -2502,6 +2513,8 @@ public class HRegion implements HeapSize { // , Writable{
         addFamilyMapToWALEdit(familyMap, walEdit);
         this.log.append(regionInfo, this.htableDescriptor.getName(),
             walEdit, clusterId, now, this.htableDescriptor);
+      } else {
+        recordPutWithoutWal(familyMap);
       }
 
       long addedSize = applyFamilyMapToMemstore(familyMap, null);
@@ -4830,14 +4843,14 @@ public class HRegion implements HeapSize { // , Writable{
   public static final long FIXED_OVERHEAD = ClassSize.align(
       ClassSize.OBJECT +
       ClassSize.ARRAY +
-      34 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
+      36 * ClassSize.REFERENCE + Bytes.SIZEOF_INT +
       (6 * Bytes.SIZEOF_LONG) +
       Bytes.SIZEOF_BOOLEAN);
 
   public static final long DEEP_OVERHEAD = FIXED_OVERHEAD +
       ClassSize.OBJECT + // closeLock
       (2 * ClassSize.ATOMIC_BOOLEAN) + // closed, closing
-      ClassSize.ATOMIC_LONG + // memStoreSize
+      (3 * ClassSize.ATOMIC_LONG) + // memStoreSize, numPutsWithoutWAL, dataInMemoryWithoutWAL 
       ClassSize.ATOMIC_INTEGER + // lockIdGenerator
       (3 * ClassSize.CONCURRENT_HASHMAP) +  // lockedRows, lockIds, scannerReadPoints
       WriteState.HEAP_SIZE + // writestate
@@ -5185,6 +5198,26 @@ public class HRegion implements HeapSize { // , Writable{
     else lock.readLock().unlock();
   }
 
+  /**
+   * Update counters for numer of puts without wal and the size of possible data loss.
+   * These information are exposed by the region server metrics.
+   */
+  private void recordPutWithoutWal(final Map<byte [], List<KeyValue>> familyMap) {
+    if (numPutsWithoutWAL.getAndIncrement() == 0) {
+      LOG.info("writing data to region " + this + 
+               " with WAL disabled. Data may be lost in the event of a crash.");
+    }
+
+    long putSize = 0;
+    for (List<KeyValue> edits : familyMap.values()) {
+      for (KeyValue kv : edits) {
+        putSize += kv.getKeyLength() + kv.getValueLength();
+      }
+    }
+
+    dataInMemoryWithoutWAL.addAndGet(putSize);
+  }
+
   /**
    * A mocked list implementaion - discards all updates.
    */

src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -1187,6 +1187,8 @@ public class HRegionServer extends RegionServer
       new HDFSBlocksDistribution();
     long totalStaticIndexSize = 0;
     long totalStaticBloomSize = 0;
+    long numPutsWithoutWAL = 0;
+    long dataInMemoryWithoutWAL = 0;
 
     // Note that this is a map of Doubles instead of Longs. This is because we
     // do effective integer division, which would perhaps truncate more than it
@@ -1199,6 +1201,8 @@ public class HRegionServer extends RegionServer
     for (Map.Entry<String, HRegion> e : this.onlineRegions.entrySet()) {
       HRegion r = e.getValue();
       memstoreSize += r.memstoreSize.get();
+      numPutsWithoutWAL += r.numPutsWithoutWAL.get();
+      dataInMemoryWithoutWAL += r.dataInMemoryWithoutWAL.get();
       readRequestsCount += r.readRequestsCount.get();
       writeRequestsCount += r.writeRequestsCount.get();
       synchronized (r.stores) {
@@ -1262,6 +1266,8 @@ public class HRegionServer extends RegionServer
     this.metrics.stores.set(stores);
     this.metrics.storefiles.set(storefiles);
     this.metrics.memstoreSizeMB.set((int) (memstoreSize / (1024 * 1024)));
+    this.metrics.mbInMemoryWithoutWAL.set((int) (dataInMemoryWithoutWAL / (1024 * 1024)));
+    this.metrics.numPutsWithoutWAL.set(numPutsWithoutWAL);
     this.metrics.storefileIndexSizeMB.set(
         (int) (storefileIndexSize / (1024 * 1024)));
     this.metrics.rootIndexSizeKB.set(

src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java

@@ -209,6 +209,18 @@ public class RegionServerMetrics implements Updater {
   public final MetricsIntValue memstoreSizeMB =
     new MetricsIntValue("memstoreSizeMB", registry);
 
+  /**
+   * Number of put with WAL disabled in this regionserver in MB
+   */
+  public final MetricsLongValue numPutsWithoutWAL =
+    new MetricsLongValue("numPutsWithoutWAL", registry);
+
+  /**
+   * Possible data loss sizes (due to put with WAL disabled) in this regionserver in MB
+   */
+  public final MetricsIntValue mbInMemoryWithoutWAL =
+    new MetricsIntValue("mbInMemoryWithoutWAL", registry);
+
   /**
    * Size of the compaction queue.
    */
@@ -366,6 +378,8 @@ public class RegionServerMetrics implements Updater {
       this.totalStaticIndexSizeKB.pushMetric(this.metricsRecord);
       this.totalStaticBloomSizeKB.pushMetric(this.metricsRecord);
       this.memstoreSizeMB.pushMetric(this.metricsRecord);
+      this.mbInMemoryWithoutWAL.pushMetric(this.metricsRecord);
+      this.numPutsWithoutWAL.pushMetric(this.metricsRecord);
       this.readRequestsCount.pushMetric(this.metricsRecord);
       this.writeRequestsCount.pushMetric(this.metricsRecord);
       this.regions.pushMetric(this.metricsRecord);
@@ -535,6 +549,10 @@ public class RegionServerMetrics implements Updater {
         Integer.valueOf(this.totalStaticBloomSizeKB.get()));
     sb = Strings.appendKeyValue(sb, this.memstoreSizeMB.getName(),
       Integer.valueOf(this.memstoreSizeMB.get()));
+    sb = Strings.appendKeyValue(sb, "mbInMemoryWithoutWAL",
+      Integer.valueOf(this.mbInMemoryWithoutWAL.get()));
+    sb = Strings.appendKeyValue(sb, "numberOfPutsWithoutWAL",
+      Long.valueOf(this.numPutsWithoutWAL.get()));
     sb = Strings.appendKeyValue(sb, "readRequestsCount",
         Long.valueOf(this.readRequestsCount.get()));
     sb = Strings.appendKeyValue(sb, "writeRequestsCount",