HubSpot Backport: HBASE-27188 Report maxStoreFileCount in jmx (#4609)

Signed-off-by: Andrew Purtell <apurtell@apache.org>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -595,6 +595,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String RPC_MUTATE_REQUEST_COUNT = "rpcMutateRequestCount";
   String RPC_MUTATE_REQUEST_COUNT_DESC =
     "Number of rpc mutation requests this RegionServer has answered.";
+  String MAX_STOREFILE_COUNT = "maxStoreFileCount";
+  String MAX_STOREFILE_COUNT_DESC = "Max store file count across all regions";
   String AVERAGE_REGION_SIZE = "averageRegionSize";
   String AVERAGE_REGION_SIZE_DESC =
       "Average region size over the RegionServer including memstore and storefile sizes.";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -90,6 +90,11 @@ public interface MetricsRegionServerWrapper {
      */
   long getNumStoreFiles();
 
+  /**
+   * Get the max number of store files across all regions of this region server.
+   */
+  long getMaxStoreFiles();
+
   /**
    * Get the size of the memstore on this region server.
    */

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java

@@ -84,6 +84,11 @@ public interface MetricsTableWrapperAggregate {
    */
   long getNumStoreFiles(String table);
 
+  /**
+   * Get the max number of store files across all regions of this table
+   */
+  long getMaxStoreFiles(String table);
+
   /**
    * @return Max age of store files for this table
    */

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -492,6 +492,8 @@ public class MetricsRegionServerSourceImpl
       .addGauge(Interns.info(WALFILE_COUNT, WALFILE_COUNT_DESC), rsWrap.getNumWALFiles())
       .addGauge(Interns.info(WALFILE_SIZE, WALFILE_SIZE_DESC), rsWrap.getWALFileSize())
       .addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles())
+      .addGauge(Interns.info(MAX_STOREFILE_COUNT, MAX_STOREFILE_COUNT_DESC),
+        rsWrap.getMaxStoreFiles())
       .addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemStoreSize())
       .addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
       .addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java

@@ -291,7 +291,12 @@ public class MetricsTableSourceImpl implements MetricsTableSource {
         mrb.addGauge(Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT,
             MetricsRegionServerSource.STOREFILE_COUNT_DESC),
           tableWrapperAgg.getNumStoreFiles(tableName.getNameAsString()));
-        mrb.addGauge(Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
+        mrb.addGauge(
+          Interns.info(tableNamePrefix + MetricsRegionServerSource.MAX_STOREFILE_COUNT,
+            MetricsRegionServerSource.MAX_STOREFILE_COUNT_DESC),
+          tableWrapperAgg.getMaxStoreFiles(tableName.getNameAsString()));
+        mrb.addGauge(
+          Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
             MetricsRegionServerSource.STOREFILE_SIZE_DESC),
             tableWrapperAgg.getStoreFileSize(tableName.getNameAsString()));
         mrb.addGauge(Interns.info(tableNamePrefix + MetricsTableSource.TABLE_SIZE,

hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java

@@ -83,6 +83,11 @@ public class MetricsTableWrapperStub implements MetricsTableWrapperAggregate {
     return 33;
   }
 
+  @Override
+  public long getMaxStoreFiles(String table) {
+    return 8;
+  }
+
   @Override
   public long getMaxStoreFileAge(String table) {
     return 44;

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -78,6 +78,7 @@ class MetricsRegionServerWrapperImpl
   private volatile long numStoreFiles = 0;
   private volatile long memstoreSize = 0;
   private volatile long storeFileSize = 0;
+  private volatile long maxStoreFileCount = 0;
   private volatile long maxStoreFileAge = 0;
   private volatile long minStoreFileAge = 0;
   private volatile long avgStoreFileAge = 0;
@@ -465,6 +466,11 @@ class MetricsRegionServerWrapperImpl
     return numStoreFiles;
   }
 
+  @Override
+  public long getMaxStoreFiles() {
+    return maxStoreFileCount;
+  }
+
   @Override
   public long getMaxStoreFileAge() {
     return maxStoreFileAge;
@@ -726,6 +732,7 @@ class MetricsRegionServerWrapperImpl
 
         long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
         long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
+        long tempMaxStoreFileCount = 0;
         long avgAgeNumerator = 0, numHFiles = 0;
         long tempMinStoreFileAge = Long.MAX_VALUE;
         long tempReadRequestsCount = 0, tempFilteredReadRequestsCount = 0,
@@ -805,6 +812,8 @@ class MetricsRegionServerWrapperImpl
             tempMemstoreSize += store.getMemStoreSize().getDataSize();
             tempStoreFileSize += store.getStorefilesSize();
 
+            tempMaxStoreFileCount = Math.max(tempMaxStoreFileCount, store.getStorefilesCount());
+
             OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
             if (storeMaxStoreFileAge.isPresent() &&
                 storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge) {
@@ -898,6 +907,7 @@ class MetricsRegionServerWrapperImpl
         numStoreFiles = tempNumStoreFiles;
         memstoreSize = tempMemstoreSize;
         storeFileSize = tempStoreFileSize;
+        maxStoreFileCount = tempMaxStoreFileCount;
         maxStoreFileAge = tempMaxStoreFileAge;
         if (regionCount > 0) {
           averageRegionSize = (memstoreSize + storeFileSize) / regionCount;

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java

@@ -77,6 +77,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr
             familyName = store.getColumnFamilyName();
 
             mt.storeFileCount += store.getStorefilesCount();
+            mt.maxStoreFileCount = Math.max(mt.maxStoreFileCount, store.getStorefilesCount());
             mt.memstoreSize += (store.getMemStoreSize().getDataSize()
                 + store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize());
             mt.storeFileSize += store.getStorefilesSize();
@@ -265,6 +266,15 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr
     return metricsTable.storeFileCount;
   }
 
+  @Override
+  public long getMaxStoreFiles(String table) {
+    MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
+    if (metricsTable == null) {
+      return 0;
+    }
+    return metricsTable.maxStoreFileCount;
+  }
+
   @Override
   public long getMaxStoreFileAge(String table) {
     MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
@@ -336,6 +346,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr
     long regionCount;
     long storeCount;
     long storeFileCount;
+    long maxStoreFileCount;
     long storeFileSize;
     long maxStoreFileAge;
     long minStoreFileAge = Long.MAX_VALUE;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -60,6 +60,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
     return 300;
   }
 
+  @Override
+  public long getMaxStoreFiles() {
+    return 23;
+  }
+
   @Override
   public long getMemStoreSize() {
     return 1025;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java

@@ -74,6 +74,7 @@ public class TestMetricsRegionServer {
     HELPER.assertGauge("regionServerStartTime", 100, serverSource);
     HELPER.assertGauge("regionCount", 101, serverSource);
     HELPER.assertGauge("storeCount", 2, serverSource);
+    HELPER.assertGauge("maxStoreFileCount", 23, serverSource);
     HELPER.assertGauge("maxStoreFileAge", 2, serverSource);
     HELPER.assertGauge("minStoreFileAge", 2, serverSource);
     HELPER.assertGauge("avgStoreFileAge", 2, serverSource);

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java

@@ -90,6 +90,7 @@ public class TestMetricsTableAggregate {
     HELPER.assertGauge(pre + "regionCount", 11, agg);
     HELPER.assertGauge(pre + "storeCount", 22, agg);
     HELPER.assertGauge(pre + "storeFileCount", 33, agg);
+    HELPER.assertGauge(pre + "maxStoreFileCount", 8, agg);
     HELPER.assertGauge(pre + "maxStoreFileAge", 44, agg);
     HELPER.assertGauge(pre + "minStoreFileAge", 55, agg);
     HELPER.assertGauge(pre + "avgStoreFileAge", 66, agg);