From 6e66e78177a8dd7a546a62abc4792f8225e0f9ca Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Mon, 11 Jul 2022 22:11:58 -0400 Subject: [PATCH] HBASE-27188 Report maxStoreFileCount in jmx (#4609) Signed-off-by: Andrew Purtell --- .../hbase/regionserver/MetricsRegionServerSource.java | 2 ++ .../regionserver/MetricsRegionServerSourceImpl.java | 2 ++ .../regionserver/MetricsRegionServerWrapper.java | 5 +++++ .../hbase/regionserver/MetricsTableSourceImpl.java | 4 ++++ .../regionserver/MetricsTableWrapperAggregate.java | 5 +++++ .../hbase/regionserver/MetricsTableWrapperStub.java | 5 +++++ .../regionserver/MetricsRegionServerWrapperImpl.java | 10 ++++++++++ .../MetricsTableWrapperAggregateImpl.java | 11 +++++++++++ .../regionserver/MetricsRegionServerWrapperStub.java | 5 +++++ .../hbase/regionserver/TestMetricsRegionServer.java | 1 + .../hbase/regionserver/TestMetricsTableAggregate.java | 1 + 11 files changed, 51 insertions(+) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index a271b1d2187..425cd9b2927 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -581,6 +581,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String RPC_MUTATE_REQUEST_COUNT = "rpcMutateRequestCount"; String RPC_MUTATE_REQUEST_COUNT_DESC = "Number of rpc mutation requests this RegionServer has answered."; + String MAX_STOREFILE_COUNT = "maxStoreFileCount"; + String MAX_STOREFILE_COUNT_DESC = "Max store file count across all regions"; String AVERAGE_REGION_SIZE = "averageRegionSize"; String AVERAGE_REGION_SIZE_DESC = "Average region size over the RegionServer including memstore and storefile sizes."; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index d264649c381..5be1ffe6bc7 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -483,6 +483,8 @@ public class MetricsRegionServerSourceImpl extends BaseSourceImpl .addGauge(Interns.info(WALFILE_COUNT, WALFILE_COUNT_DESC), rsWrap.getNumWALFiles()) .addGauge(Interns.info(WALFILE_SIZE, WALFILE_SIZE_DESC), rsWrap.getWALFileSize()) .addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles()) + .addGauge(Interns.info(MAX_STOREFILE_COUNT, MAX_STOREFILE_COUNT_DESC), + rsWrap.getMaxStoreFiles()) .addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemStoreSize()) .addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize()) .addGauge(Interns.info(STOREFILE_SIZE_GROWTH_RATE, STOREFILE_SIZE_GROWTH_RATE_DESC), diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index d4f33737c44..7367897711f 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -91,6 +91,11 @@ public interface MetricsRegionServerWrapper { */ long getNumStoreFiles(); + /** + * Get the max number of store files across all regions of this region server. + */ + long getMaxStoreFiles(); + /** * Get the size of the memstore on this region server. */ diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java index 19c6d845ceb..3dc1dc8fc66 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java @@ -299,6 +299,10 @@ public class MetricsTableSourceImpl implements MetricsTableSource { Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT, MetricsRegionServerSource.STOREFILE_COUNT_DESC), tableWrapperAgg.getNumStoreFiles(tableName.getNameAsString())); + mrb.addGauge( + Interns.info(tableNamePrefix + MetricsRegionServerSource.MAX_STOREFILE_COUNT, + MetricsRegionServerSource.MAX_STOREFILE_COUNT_DESC), + tableWrapperAgg.getMaxStoreFiles(tableName.getNameAsString())); mrb.addGauge( Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE, MetricsRegionServerSource.STOREFILE_SIZE_DESC), diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java index 284fb57cd23..066628f5a74 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java @@ -88,6 +88,11 @@ public interface MetricsTableWrapperAggregate { */ long getNumStoreFiles(String table); + /** + * Get the max number of store files across all regions of this table + */ + long getMaxStoreFiles(String table); + /** * @return Max age of store files for this table */ diff --git a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java index 029b75e52cf..b8f4a9a6167 100644 --- a/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java +++ b/hbase-hadoop-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java @@ -82,6 +82,11 @@ public class MetricsTableWrapperStub implements MetricsTableWrapperAggregate { return 33; } + @Override + public long getMaxStoreFiles(String table) { + return 8; + } + @Override public long getMaxStoreFileAge(String table) { return 44; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index ae5e904955e..2bd26ae6497 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -82,6 +82,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { private volatile long offHeapMemstoreSize = 0; private volatile long storeFileSize = 0; private volatile double storeFileSizeGrowthRate = 0; + private volatile long maxStoreFileCount = 0; private volatile long maxStoreFileAge = 0; private volatile long minStoreFileAge = 0; private volatile long avgStoreFileAge = 0; @@ -480,6 +481,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { return numStoreFiles; } + @Override + public long getMaxStoreFiles() { + return maxStoreFileCount; + } + @Override public long getMaxStoreFileAge() { return maxStoreFileAge; @@ -763,6 +769,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { long tempNumStores = 0, tempNumStoreFiles = 0, tempStoreFileSize = 0; long tempMemstoreSize = 0, tempOnHeapMemstoreSize = 0, tempOffHeapMemstoreSize = 0; long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0; + long tempMaxStoreFileCount = 0; long avgAgeNumerator = 0, numHFiles = 0; long tempMinStoreFileAge = Long.MAX_VALUE; long tempFilteredReadRequestsCount = 0, tempCpRequestsCount = 0; @@ -846,6 +853,8 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { tempOffHeapMemstoreSize += store.getMemStoreSize().getOffHeapSize(); tempStoreFileSize += store.getStorefilesSize(); + tempMaxStoreFileCount = Math.max(tempMaxStoreFileCount, store.getStorefilesCount()); + OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge(); if ( storeMaxStoreFileAge.isPresent() @@ -952,6 +961,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { onHeapMemstoreSize = tempOnHeapMemstoreSize; offHeapMemstoreSize = tempOffHeapMemstoreSize; storeFileSize = tempStoreFileSize; + maxStoreFileCount = tempMaxStoreFileCount; maxStoreFileAge = tempMaxStoreFileAge; if (regionCount > 0) { averageRegionSize = (memstoreSize + storeFileSize) / regionCount; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java index 8a367594239..37ba794a3ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java @@ -75,6 +75,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr familyName = store.getColumnFamilyName(); mt.storeFileCount += store.getStorefilesCount(); + mt.maxStoreFileCount = Math.max(mt.maxStoreFileCount, store.getStorefilesCount()); mt.memstoreSize += (store.getMemStoreSize().getDataSize() + store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize()); mt.storeFileSize += store.getStorefilesSize(); @@ -263,6 +264,15 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr return metricsTable.storeFileCount; } + @Override + public long getMaxStoreFiles(String table) { + MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table)); + if (metricsTable == null) { + return 0; + } + return metricsTable.maxStoreFileCount; + } + @Override public long getMaxStoreFileAge(String table) { MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table)); @@ -334,6 +344,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr long regionCount; long storeCount; long storeFileCount; + long maxStoreFileCount; long storeFileSize; long maxStoreFileAge; long minStoreFileAge = Long.MAX_VALUE; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index d604cf00d49..1d6113c0797 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -62,6 +62,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe return 300; } + @Override + public long getMaxStoreFiles() { + return 23; + } + @Override public long getMemStoreSize() { return 1025; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index a44f3c6b62e..2efe3128a83 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -72,6 +72,7 @@ public class TestMetricsRegionServer { HELPER.assertGauge("regionServerStartTime", 100, serverSource); HELPER.assertGauge("regionCount", 101, serverSource); HELPER.assertGauge("storeCount", 2, serverSource); + HELPER.assertGauge("maxStoreFileCount", 23, serverSource); HELPER.assertGauge("maxStoreFileAge", 2, serverSource); HELPER.assertGauge("minStoreFileAge", 2, serverSource); HELPER.assertGauge("avgStoreFileAge", 2, serverSource); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java index b77d9855557..169f7c54c99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java @@ -91,6 +91,7 @@ public class TestMetricsTableAggregate { HELPER.assertGauge(pre + "regionCount", 11, agg); HELPER.assertGauge(pre + "storeCount", 22, agg); HELPER.assertGauge(pre + "storeFileCount", 33, agg); + HELPER.assertGauge(pre + "maxStoreFileCount", 8, agg); HELPER.assertGauge(pre + "maxStoreFileAge", 44, agg); HELPER.assertGauge(pre + "minStoreFileAge", 55, agg); HELPER.assertGauge(pre + "avgStoreFileAge", 66, agg);