From be59eb7e8d404b2523862424a5ed1ce1dbf7f2e1 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Fri, 4 Mar 2022 17:03:32 -0500 Subject: [PATCH] HBASE-26731 Add metrics for active and expired scanners (#4145) Signed-off-by: Andrew Purtell --- .../regionserver/MetricsRegionServerSource.java | 8 ++++++++ .../regionserver/MetricsRegionServerSourceImpl.java | 13 ++++++++++++- .../regionserver/MetricsRegionServerWrapper.java | 2 ++ .../hbase/regionserver/MetricsRegionServer.java | 4 ++++ .../MetricsRegionServerWrapperImpl.java | 5 +++++ .../hadoop/hbase/regionserver/RSRpcServices.java | 1 + .../MetricsRegionServerWrapperStub.java | 5 +++++ .../hbase/regionserver/TestMetricsRegionServer.java | 8 ++++++++ 8 files changed, 45 insertions(+), 1 deletion(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 9d880d16137..97896d41022 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -226,6 +226,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo */ void updateCompactionOutputSize(boolean isMajor, long bytes); + void incrScannerLeaseExpired(); + // Strings used for exporting to metrics system. String REGION_COUNT = "regionCount"; String REGION_COUNT_DESC = "Number of regions"; @@ -598,4 +600,10 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String BYTE_BUFF_ALLOCATOR_TOTAL_BUFFER_COUNT_DESC = "Total buffer count in ByteBuffAllocator"; String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT = "ByteBuffAllocatorUsedBufferCount"; String BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC = "Used buffer count in ByteBuffAllocator"; + + String ACTIVE_SCANNERS = "activeScanners"; + String ACTIVE_SCANNERS_DESC = "Gauge of currently active scanners"; + + String SCANNER_LEASE_EXPIRED_COUNT = "scannerLeaseExpiredCount"; + String SCANNER_LEASE_EXPIRED_COUNT_DESC = "Count of scanners which were expired due to scanner lease timeout"; } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index 7b0225482cb..966d75ac9fc 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -91,6 +91,8 @@ public class MetricsRegionServerSourceImpl private final MetricHistogram pausesWithGc; private final MetricHistogram pausesWithoutGc; + private final MutableFastCounter scannerLeaseExpiredCount; + public MetricsRegionServerSourceImpl(MetricsRegionServerWrapper rsWrap) { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT, rsWrap); } @@ -179,6 +181,8 @@ public class MetricsRegionServerSourceImpl WARN_THRESHOLD_COUNT_DESC, 0L); pausesWithGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITH_GC_KEY); pausesWithoutGc = getMetricsRegistry().newTimeHistogram(PAUSE_TIME_WITHOUT_GC_KEY); + + scannerLeaseExpiredCount = getMetricsRegistry().newCounter(SCANNER_LEASE_EXPIRED_COUNT, SCANNER_LEASE_EXPIRED_COUNT_DESC, 0L); } @Override @@ -322,6 +326,11 @@ public class MetricsRegionServerSourceImpl } } + @Override + public void incrScannerLeaseExpired() { + scannerLeaseExpiredCount.incr(); + } + /** * Yes this is a get function that doesn't return anything. Thanks Hadoop for breaking all * expectations of java programmers. Instead of returning anything Hadoop metrics expects @@ -586,7 +595,9 @@ public class MetricsRegionServerSourceImpl rsWrap.getByteBuffAllocatorTotalBufferCount()) .addGauge(Interns.info(BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT, BYTE_BUFF_ALLOCATOR_USED_BUFFER_COUNT_DESC), - rsWrap.getByteBuffAllocatorUsedBufferCount()); + rsWrap.getByteBuffAllocatorUsedBufferCount()) + .addGauge(Interns.info(ACTIVE_SCANNERS, ACTIVE_SCANNERS_DESC), + rsWrap.getActiveScanners()); } @Override diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 32c73d65321..b424cdb21db 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -569,4 +569,6 @@ public interface MetricsRegionServerWrapper { long getByteBuffAllocatorTotalBufferCount(); long getByteBuffAllocatorUsedBufferCount(); + + int getActiveScanners(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java index 07a90f6db95..0dc835fcd3c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServer.java @@ -317,4 +317,8 @@ public class MetricsRegionServer { serverWriteQueryMeter.mark(); } } + + public void incrScannerLeaseExpired() { + serverSource.incrScannerLeaseExpired(); + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 8802715248c..517a4cbcd11 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -672,6 +672,11 @@ class MetricsRegionServerWrapperImpl return mobFileCacheHitRatio * 100; } + @Override + public int getActiveScanners() { + return regionServer.getRpcServices().getScannersCount(); + } + /** * This is the runnable that will be executed on the executor every PERIOD number of seconds * It will take metrics/numbers from all of the regions and use them to compute point in diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java index 5b57ac18a7c..a23b22fffeb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RSRpcServices.java @@ -478,6 +478,7 @@ public class RSRpcServices extends HBaseRpcServicesBase return; } LOG.info("Scanner lease {} expired {}", this.scannerName, rsh); + server.getMetrics().incrScannerLeaseExpired(); RegionScanner s = rsh.s; HRegion region = null; try { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index ae9666dd303..a874cb17181 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -143,6 +143,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe return 0; } + @Override + public int getActiveScanners() { + return 0; + } + @Override public long getReadRequestsCount() { return 997; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index 3663f85dd78..f2e485442bd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -250,6 +250,14 @@ public class TestMetricsRegionServer { HELPER.assertCounter("pauseTimeWithGc_num_ops", 1, serverSource); } + @Test + public void testScannerMetrics() { + HELPER.assertCounter("scannerLeaseExpiredCount", 0, serverSource); + rsm.incrScannerLeaseExpired(); + HELPER.assertCounter("scannerLeaseExpiredCount", 1, serverSource); + HELPER.assertGauge("activeScanners", 0, serverSource); + } + @Test public void testTableQueryMeterSwitch() { TableName tn1 = TableName.valueOf("table1");