HBASE-26731 Add metrics for active and expired scanners (#4145)

Signed-off-by: Andrew Purtell <apurtell@apache.org>

Conflicts:
	hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java
	hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -226,6 +226,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
    */
   void updateCompactionOutputSize(boolean isMajor, long bytes);
 
+  void incrScannerLeaseExpired();
+
   // Strings used for exporting to metrics system.
   String REGION_COUNT = "regionCount";
   String REGION_COUNT_DESC = "Number of regions";
@@ -592,4 +594,10 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String BYTE_BUFF_ALLOCATOR_TOTAL_BUFFER_COUNT_DESC = "Total buffer count in ByteBuffAllocator";
   String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT = "ByteBuffAllocatorUsedBufferCount";
   String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC = "Used buffer count in ByteBuffAllocator";
+
+  String ACTIVE_SCANNERS = "activeScanners";
+  String ACTIVE_SCANNERS_DESC = "Gauge of currently active scanners";
+
+  String SCANNER_LEASE_EXPIRED_COUNT = "scannerLeaseExpiredCount";
+  String SCANNER_LEASE_EXPIRED_COUNT_DESC = "Count of scanners which were expired due to scanner lease timeout";
 }

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -559,4 +559,6 @@ public interface MetricsRegionServerWrapper {
   long getByteBuffAllocatorTotalBufferCount();
 
   long getByteBuffAllocatorUsedBufferCount();
+
+  int getActiveScanners();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -91,6 +91,8 @@ public class MetricsRegionServerSourceImpl
   private final MetricHistogram pausesWithGc;
   private final MetricHistogram pausesWithoutGc;
 
+  private final MutableFastCounter scannerLeaseExpiredCount;
+
   public MetricsRegionServerSourceImpl(MetricsRegionServerWrapper rsWrap) {
     this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, rsWrap);
   }
@@ -179,6 +181,8 @@ public class MetricsRegionServerSourceImpl
       WARN_THRESHOLD_COUNT_DESC, 0L);
     pausesWithGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITH_GC_KEY);
     pausesWithoutGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITHOUT_GC_KEY);
+
+    scannerLeaseExpiredCount = getMetricsRegistry().newCounter(SCANNER_LEASE_EXPIRED_COUNT, SCANNER_LEASE_EXPIRED_COUNT_DESC, 0L);
   }
 
   @Override
@@ -322,6 +326,11 @@ public class MetricsRegionServerSourceImpl
     }
   }
 
+  @Override
+  public void incrScannerLeaseExpired() {
+    scannerLeaseExpiredCount.incr();
+  }
+
   /**
    * Yes this is a get function that doesn't return anything.  Thanks Hadoop for breaking all
    * expectations of java programmers.  Instead of returning anything Hadoop metrics expects
@@ -582,7 +591,9 @@ public class MetricsRegionServerSourceImpl
                 rsWrap.getByteBuffAllocatorTotalBufferCount())
             .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT,
                 BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC),
-                rsWrap.getByteBuffAllocatorUsedBufferCount());
+                rsWrap.getByteBuffAllocatorUsedBufferCount())
+            .addGauge(Interns.info(ACTIVE_SCANNERS, ACTIVE_SCANNERS_DESC),
+                rsWrap.getActiveScanners());
   }
 
   @Override

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java

@@ -317,4 +317,8 @@ public class MetricsRegionServer {
   public void incrementRegionSizeReportingChoreTime(long time) {
     quotaSource.incrementRegionSizeReportingChoreTime(time);
   }
+
+  public void incrScannerLeaseExpired() {
+    serverSource.incrScannerLeaseExpired();
+  }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -661,6 +661,11 @@ class MetricsRegionServerWrapperImpl
     return mobFileCacheHitRatio * 100;
   }
 
+  @Override
+  public int getActiveScanners() {
+    return regionServer.getRpcServices().getScannersCount();
+  }
+
   /**
    * This is the runnable that will be executed on the executor every PERIOD number of seconds
    * It will take metrics/numbers from all of the regions and use them to compute point in

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java

@@ -534,6 +534,7 @@ public class RSRpcServices implements HBaseRPCErrorHandler, AdminService.Blockin
         return;
       }
       LOG.info("Scanner lease {} expired {}", this.scannerName, rsh);
+      server.getMetrics().incrScannerLeaseExpired();
       RegionScanner s = rsh.s;
       HRegion region = null;
       try {

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -138,6 +138,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
     return 0;
   }
 
+  @Override
+  public int getActiveScanners() {
+    return 0;
+  }
+
   @Override
   public long getReadRequestsCount() {
     return 997;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java

@@ -251,7 +251,7 @@ public class TestMetricsRegionServer {
   }
 
   @Test
-  public void testServerQueryMeterSwitch() {
+  public void testTableQueryMeterSwitch() {
     TableName tn1 = TableName.valueOf("table1");
     // has been set disable in setUp()
     rsm.updateReadQueryMeter(tn1, 500L);
@@ -271,5 +271,14 @@ public class TestMetricsRegionServer {
     rsm.updateWriteQueryMeter(tn1, 500L);
     HELPER.assertGauge("ServerWriteQueryPerSecond_count", 500L, serverSource);
   }
+
+  @Test
+  public void testScannerMetrics() {
+    HELPER.assertCounter("scannerLeaseExpiredCount", 0, serverSource);
+    rsm.incrScannerLeaseExpired();
+    HELPER.assertCounter("scannerLeaseExpiredCount", 1, serverSource);
+    HELPER.assertGauge("activeScanners", 0, serverSource);
+  }
+
 }