HBASE-18469 Correct RegionServer metric of totalRequestCount

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -244,6 +244,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String TOTAL_REQUEST_COUNT = "totalRequestCount";
   String TOTAL_REQUEST_COUNT_DESC =
       "Total number of requests this RegionServer has answered.";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT = "totalRowActionRequestCount";
+  String TOTAL_ROW_ACTION_REQUEST_COUNT_DESC =
+      "Total number of region requests this RegionServer has answered, count by row-level action";
   String READ_REQUEST_COUNT = "readRequestCount";
   String READ_REQUEST_COUNT_DESC =
       "Number of read requests this region server has answered.";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -457,4 +457,6 @@ public interface MetricsRegionServerWrapper {
   long getDeleteFamilyBloomHitCount();
 
   long getTrailerHitCount();
+
+  long getTotalRowActionRequestCount();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -354,6 +354,8 @@ public class MetricsRegionServerSourceImpl
           .addGauge(Interns.info(AVERAGE_REGION_SIZE, AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize())
           .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC),
               rsWrap.getTotalRequestCount())
+          .addCounter(Interns.info(TOTAL_ROW_ACTION_REQUEST_COUNT, TOTAL_ROW_ACTION_REQUEST_COUNT_DESC),
+              rsWrap.getTotalRowActionRequestCount())
           .addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC),
               rsWrap.getReadRequestsCount())
           .addCounter(Interns.info(FILTERED_READ_REQUEST_COUNT, FILTERED_READ_REQUEST_COUNT_DESC),

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -218,6 +218,11 @@ class MetricsRegionServerWrapperImpl
     return regionServer.rpcServices.requestCount.sum();
   }
 
+  @Override
+  public long getTotalRowActionRequestCount() {
+    return regionServer.rpcServices.requestRowActionCount.sum();
+  }
+
   @Override
   public int getSplitQueueSize() {
     if (this.regionServer.compactSplitThread == null) {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java

@@ -256,8 +256,13 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
   static final int BATCH_ROWS_THRESHOLD_DEFAULT = 5000;
 
   // Request counter. (Includes requests that are not serviced by regions.)
+  // Count only once for requests with multiple actions like multi/caching-scan/replayBatch
   final LongAdder requestCount = new LongAdder();
 
+  // Request counter. (Excludes requests that are not serviced by regions.)
+  // Count rows for requests with multiple actions like multi/caching-scan/replayBatch
+  final LongAdder requestRowActionCount = new LongAdder();
+
   // Request counter for rpc get
   final LongAdder rpcGetRequestCount = new LongAdder();
 
@@ -1091,7 +1096,8 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
           it.remove();
         }
       }
-      requestCount.add(mutations.size());
+      requestCount.increment();
+      requestRowActionCount.add(mutations.size());
       if (!region.getRegionInfo().isMetaTable()) {
         regionServer.cacheFlusher.reclaimMemStoreMemory();
       }
@@ -2393,6 +2399,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
     try {
       checkOpen();
       requestCount.increment();
+      requestRowActionCount.increment();
       rpcGetRequestCount.increment();
       Region region = getRegion(request.getRegion());
 
@@ -2549,11 +2556,12 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
     RegionScannersCloseCallBack closeCallBack = null;
     RpcCallContext context = RpcServer.getCurrentCall();
     this.rpcMultiRequestCount.increment();
+    this.requestCount.increment();
     Map<RegionSpecifier, ClientProtos.RegionLoadStats> regionStats = new HashMap<>(request
       .getRegionActionCount());
     ActivePolicyEnforcement spaceQuotaEnforcement = getSpaceQuotaManager().getActiveEnforcements();
     for (RegionAction regionAction : request.getRegionActionList()) {
-      this.requestCount.add(regionAction.getActionCount());
+      this.requestRowActionCount.add(regionAction.getActionCount());
       OperationQuota quota;
       Region region;
       regionActionResultBuilder.clear();
@@ -2687,6 +2695,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
     try {
       checkOpen();
       requestCount.increment();
+      requestRowActionCount.increment();
       rpcMutateRequestCount.increment();
       Region region = getRegion(request.getRegion());
       MutateResponse.Builder builder = MutateResponse.newBuilder();
@@ -3133,6 +3142,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler,
         }
       }
       region.updateReadRequestsCount(numOfResults);
+      requestRowActionCount.add(numOfResults);
       long end = EnvironmentEdgeManager.currentTime();
       long responseCellSize = context != null ? context.getResponseCellSize() : 0;
       region.getMetrics().updateScanTime(end - before);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -100,6 +100,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
     return 899;
   }
 
+  @Override
+  public long getTotalRowActionRequestCount() {
+    return getReadRequestsCount() + getWriteRequestsCount();
+  }
+
   @Override
   public long getReadRequestsCount() {
     return 997;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java

@@ -72,6 +72,10 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("memstoreSize", 1025, serverSource);
     HELPER.assertGauge("storeFileSize", 1900, serverSource);
     HELPER.assertCounter("totalRequestCount", 899, serverSource);
+    HELPER.assertCounter("totalRowActionRequestCount",
+      HELPER.getCounter("readRequestCount", serverSource)
+          + HELPER.getCounter("writeRequestCount", serverSource),
+      serverSource);
     HELPER.assertCounter("readRequestCount", 997, serverSource);
     HELPER.assertCounter("filteredReadRequestCount", 1997, serverSource);
     HELPER.assertCounter("writeRequestCount", 707, serverSource);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java

@@ -224,6 +224,21 @@ public class TestRegionServerMetrics {
     }
   }
 
+  public void doScan(int n, boolean caching) throws IOException {
+    Scan scan = new Scan();
+    if (caching) {
+      scan.setCaching(n);
+    } else {
+      scan.setCaching(1);
+    }
+    ResultScanner scanner = table.getScanner(scan);
+    for (int i = 0; i < n; i++) {
+      Result res = scanner.next();
+      LOG.debug(
+        "Result row: " + Bytes.toString(res.getRow()) + ", value: " + res.getValue(cf, qualifier));
+    }
+  }
+
   @Test
   public void testRegionCount() throws Exception {
     metricsHelper.assertGauge("regionCount", 1, serverSource);
@@ -242,6 +257,7 @@ public class TestRegionServerMetrics {
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
     long requests = metricsHelper.getCounter("totalRequestCount", serverSource);
+    long rowActionRequests = metricsHelper.getCounter("totalRowActionRequestCount", serverSource);
     long readRequests = metricsHelper.getCounter("readRequestCount", serverSource);
     long writeRequests = metricsHelper.getCounter("writeRequestCount", serverSource);
 
@@ -249,6 +265,7 @@ public class TestRegionServerMetrics {
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
     assertCounter("totalRequestCount", requests + 30);
+    assertCounter("totalRowActionRequestCount", rowActionRequests + 30);
     assertCounter("readRequestCount", readRequests);
     assertCounter("writeRequestCount", writeRequests + 30);
 
@@ -256,6 +273,7 @@ public class TestRegionServerMetrics {
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
     assertCounter("totalRequestCount", requests + 40);
+    assertCounter("totalRowActionRequestCount", rowActionRequests + 40);
     assertCounter("readRequestCount", readRequests + 10);
     assertCounter("writeRequestCount", writeRequests + 30);
 
@@ -265,16 +283,34 @@ public class TestRegionServerMetrics {
     doNGets(10, true);  // true = batch
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
-    assertCounter("totalRequestCount", requests + 50);
+    assertCounter("totalRequestCount", requests + 41);
+    assertCounter("totalRowActionRequestCount", rowActionRequests + 50);
     assertCounter("readRequestCount", readRequests + 20);
     assertCounter("writeRequestCount", writeRequests + 30);
 
     doNPuts(30, true);
 
     metricsRegionServer.getRegionServerWrapper().forceRecompute();
-    assertCounter("totalRequestCount", requests + 80);
+    assertCounter("totalRequestCount", requests + 42);
+    assertCounter("totalRowActionRequestCount", rowActionRequests + 80);
     assertCounter("readRequestCount", readRequests + 20);
     assertCounter("writeRequestCount", writeRequests + 60);
+
+    doScan(10, false); // test after batch put so we have enough lines
+    metricsRegionServer.getRegionServerWrapper().forceRecompute();
+    assertCounter("totalRequestCount", requests + 52);
+    assertCounter("totalRowActionRequestCount", rowActionRequests + 90);
+    assertCounter("readRequestCount", readRequests + 30);
+    assertCounter("writeRequestCount", writeRequests + 60);
+    numScanNext += 10;
+
+    doScan(10, true); // true = caching
+    metricsRegionServer.getRegionServerWrapper().forceRecompute();
+    assertCounter("totalRequestCount", requests + 53);
+    assertCounter("totalRowActionRequestCount", rowActionRequests + 100);
+    assertCounter("readRequestCount", readRequests + 40);
+    assertCounter("writeRequestCount", writeRequests + 60);
+    numScanNext += 1;
   }
 
   @Test