From 7dfed53819f5a81517d54aa306107359db123487 Mon Sep 17 00:00:00 2001 From: Bryan Beaudreault Date: Mon, 11 Jul 2022 22:11:58 -0400 Subject: [PATCH] HBASE-27188 Report maxStoreFileCount in jmx (#4609) Signed-off-by: Andrew Purtell --- .../hbase/regionserver/MetricsRegionServerSource.java | 2 ++ .../regionserver/MetricsRegionServerWrapper.java | 5 +++++ .../regionserver/MetricsTableWrapperAggregate.java | 5 +++++ .../regionserver/MetricsRegionServerSourceImpl.java | 2 ++ .../hbase/regionserver/MetricsTableSourceImpl.java | 4 ++++ .../hbase/regionserver/MetricsTableWrapperStub.java | 5 +++++ .../regionserver/MetricsRegionServerWrapperImpl.java | 10 ++++++++++ .../MetricsTableWrapperAggregateImpl.java | 11 +++++++++++ .../regionserver/MetricsRegionServerWrapperStub.java | 5 +++++ .../hbase/regionserver/TestMetricsRegionServer.java | 1 + .../hbase/regionserver/TestMetricsTableAggregate.java | 1 + 11 files changed, 51 insertions(+) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 7b34b549560..2bb72bef1ae 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -572,6 +572,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo String RPC_MUTATE_REQUEST_COUNT = "rpcMutateRequestCount"; String RPC_MUTATE_REQUEST_COUNT_DESC = "Number of rpc mutation requests this RegionServer has answered."; + String MAX_STOREFILE_COUNT = "maxStoreFileCount"; + String MAX_STOREFILE_COUNT_DESC = "Max store file count across all regions"; String AVERAGE_REGION_SIZE = "averageRegionSize"; String AVERAGE_REGION_SIZE_DESC = "Average region size over the RegionServer including memstore and storefile sizes."; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index d0c1c4a3102..ed446034d6c 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -85,6 +85,11 @@ public interface MetricsRegionServerWrapper { */ long getNumStoreFiles(); + /** + * Get the max number of store files across all regions of this region server. + */ + long getMaxStoreFiles(); + /** * Get the size of the memstore on this region server. */ diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java index ad72876f02e..95489cb10b0 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java @@ -83,6 +83,11 @@ public interface MetricsTableWrapperAggregate { */ long getNumStoreFiles(String table); + /** + * Get the max number of store files across all regions of this table + */ + long getMaxStoreFiles(String table); + /** * @return Max age of store files for this table */ diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index bec409c8eee..1d81fbb78cb 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -471,6 +471,8 @@ public class MetricsRegionServerSourceImpl extends BaseSourceImpl .addGauge(Interns.info(WALFILE_COUNT, WALFILE_COUNT_DESC), rsWrap.getNumWALFiles()) .addGauge(Interns.info(WALFILE_SIZE, WALFILE_SIZE_DESC), rsWrap.getWALFileSize()) .addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles()) + .addGauge(Interns.info(MAX_STOREFILE_COUNT, MAX_STOREFILE_COUNT_DESC), + rsWrap.getMaxStoreFiles()) .addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemStoreSize()) .addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize()) .addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC), diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java index 65f8f04102e..afc4310fb69 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java @@ -295,6 +295,10 @@ public class MetricsTableSourceImpl implements MetricsTableSource { Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT, MetricsRegionServerSource.STOREFILE_COUNT_DESC), tableWrapperAgg.getNumStoreFiles(tableName.getNameAsString())); + mrb.addGauge( + Interns.info(tableNamePrefix + MetricsRegionServerSource.MAX_STOREFILE_COUNT, + MetricsRegionServerSource.MAX_STOREFILE_COUNT_DESC), + tableWrapperAgg.getMaxStoreFiles(tableName.getNameAsString())); mrb.addGauge( Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE, MetricsRegionServerSource.STOREFILE_SIZE_DESC), diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java index 9fcbd24f9ea..c1d42718ef6 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java @@ -82,6 +82,11 @@ public class MetricsTableWrapperStub implements MetricsTableWrapperAggregate { return 33; } + @Override + public long getMaxStoreFiles(String table) { + return 8; + } + @Override public long getMaxStoreFileAge(String table) { return 44; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 4265633e899..713b76abec5 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -76,6 +76,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { private volatile long numStoreFiles = 0; private volatile long memstoreSize = 0; private volatile long storeFileSize = 0; + private volatile long maxStoreFileCount = 0; private volatile long maxStoreFileAge = 0; private volatile long minStoreFileAge = 0; private volatile long avgStoreFileAge = 0; @@ -462,6 +463,11 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { return numStoreFiles; } + @Override + public long getMaxStoreFiles() { + return maxStoreFileCount; + } + @Override public long getMaxStoreFileAge() { return maxStoreFileAge; @@ -718,6 +724,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0; long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0; + long tempMaxStoreFileCount = 0; long avgAgeNumerator = 0, numHFiles = 0; long tempMinStoreFileAge = Long.MAX_VALUE; long tempReadRequestsCount = 0, tempFilteredReadRequestsCount = 0, @@ -797,6 +804,8 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { tempMemstoreSize += store.getMemStoreSize().getDataSize(); tempStoreFileSize += store.getStorefilesSize(); + tempMaxStoreFileCount = Math.max(tempMaxStoreFileCount, store.getStorefilesCount()); + OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge(); if ( storeMaxStoreFileAge.isPresent() @@ -894,6 +903,7 @@ class MetricsRegionServerWrapperImpl implements MetricsRegionServerWrapper { numStoreFiles = tempNumStoreFiles; memstoreSize = tempMemstoreSize; storeFileSize = tempStoreFileSize; + maxStoreFileCount = tempMaxStoreFileCount; maxStoreFileAge = tempMaxStoreFileAge; if (regionCount > 0) { averageRegionSize = (memstoreSize + storeFileSize) / regionCount; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java index 6e9d862dcf4..c5ea89c4a9f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java @@ -75,6 +75,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr familyName = store.getColumnFamilyName(); mt.storeFileCount += store.getStorefilesCount(); + mt.maxStoreFileCount = Math.max(mt.maxStoreFileCount, store.getStorefilesCount()); mt.memstoreSize += (store.getMemStoreSize().getDataSize() + store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize()); mt.storeFileSize += store.getStorefilesSize(); @@ -262,6 +263,15 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr return metricsTable.storeFileCount; } + @Override + public long getMaxStoreFiles(String table) { + MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table)); + if (metricsTable == null) { + return 0; + } + return metricsTable.maxStoreFileCount; + } + @Override public long getMaxStoreFileAge(String table) { MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table)); @@ -333,6 +343,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr long regionCount; long storeCount; long storeFileCount; + long maxStoreFileCount; long storeFileSize; long maxStoreFileAge; long minStoreFileAge = Long.MAX_VALUE; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 38fda71a161..4fe7d3d6f0b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -59,6 +59,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe return 300; } + @Override + public long getMaxStoreFiles() { + return 23; + } + @Override public long getMemStoreSize() { return 1025; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java index e3f2fe2059b..926ac834360 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java @@ -74,6 +74,7 @@ public class TestMetricsRegionServer { HELPER.assertGauge("regionServerStartTime", 100, serverSource); HELPER.assertGauge("regionCount", 101, serverSource); HELPER.assertGauge("storeCount", 2, serverSource); + HELPER.assertGauge("maxStoreFileCount", 23, serverSource); HELPER.assertGauge("maxStoreFileAge", 2, serverSource); HELPER.assertGauge("minStoreFileAge", 2, serverSource); HELPER.assertGauge("avgStoreFileAge", 2, serverSource); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java index b77d9855557..169f7c54c99 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java @@ -91,6 +91,7 @@ public class TestMetricsTableAggregate { HELPER.assertGauge(pre + "regionCount", 11, agg); HELPER.assertGauge(pre + "storeCount", 22, agg); HELPER.assertGauge(pre + "storeFileCount", 33, agg); + HELPER.assertGauge(pre + "maxStoreFileCount", 8, agg); HELPER.assertGauge(pre + "maxStoreFileAge", 44, agg); HELPER.assertGauge(pre + "minStoreFileAge", 55, agg); HELPER.assertGauge(pre + "avgStoreFileAge", 66, agg);