HBASE-23590 : Update maxStoreFileRefCount to maxCompactedStoreFileRefCount for auto region recovery based on old reader references

Signed-off-by: Anoop Sam John <anoopsamjohn@apache.org>

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java

@@ -366,8 +366,8 @@ public class RegionLoad implements RegionMetrics {
   }
 
   @Override
-  public int getMaxStoreFileRefCount() {
+  public int getMaxCompactedStoreFileRefCount() {
-    return metrics.getMaxStoreFileRefCount();
+    return metrics.getMaxCompactedStoreFileRefCount();
   }
 
   /**

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java

@@ -150,8 +150,8 @@ public interface RegionMetrics {
   int getStoreRefCount();
 
   /**
-   * @return the max reference count for any store file among all stores files
+   * @return the max reference count for any store file among all compacted stores files
    *   of this region
    */
-  int getMaxStoreFileRefCount();
+  int getMaxCompactedStoreFileRefCount();
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java

@@ -65,7 +65,7 @@ public final class RegionMetricsBuilder {
         .setStoreCount(regionLoadPB.getStores())
         .setStoreFileCount(regionLoadPB.getStorefiles())
         .setStoreRefCount(regionLoadPB.getStoreRefCount())
-        .setMaxStoreFileRefCount(regionLoadPB.getMaxStoreFileRefCount())
+        .setMaxCompactedStoreFileRefCount(regionLoadPB.getMaxCompactedStoreFileRefCount())
         .setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(), Size.Unit.MEGABYTE))
         .setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream()
           .collect(Collectors.toMap(
@@ -112,7 +112,7 @@ public final class RegionMetricsBuilder {
         .setStores(regionMetrics.getStoreCount())
         .setStorefiles(regionMetrics.getStoreFileCount())
         .setStoreRefCount(regionMetrics.getStoreRefCount())
-        .setMaxStoreFileRefCount(regionMetrics.getMaxStoreFileRefCount())
+        .setMaxCompactedStoreFileRefCount(regionMetrics.getMaxCompactedStoreFileRefCount())
         .setStorefileSizeMB((int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE))
         .addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId()))
         .setStoreUncompressedSizeMB(
@@ -128,7 +128,7 @@ public final class RegionMetricsBuilder {
   private int storeCount;
   private int storeFileCount;
   private int storeRefCount;
-  private int maxStoreFileRefCount;
+  private int maxCompactedStoreFileRefCount;
   private long compactingCellCount;
   private long compactedCellCount;
   private Size storeFileSize = Size.ZERO;
@@ -161,8 +161,8 @@ public final class RegionMetricsBuilder {
     this.storeRefCount = value;
     return this;
   }
-  public RegionMetricsBuilder setMaxStoreFileRefCount(int value) {
+  public RegionMetricsBuilder setMaxCompactedStoreFileRefCount(int value) {
-    this.maxStoreFileRefCount = value;
+    this.maxCompactedStoreFileRefCount = value;
     return this;
   }
   public RegionMetricsBuilder setCompactingCellCount(long value) {
@@ -235,7 +235,7 @@ public final class RegionMetricsBuilder {
         storeCount,
         storeFileCount,
         storeRefCount,
-        maxStoreFileRefCount,
+        maxCompactedStoreFileRefCount,
         compactingCellCount,
         compactedCellCount,
         storeFileSize,
@@ -259,7 +259,7 @@ public final class RegionMetricsBuilder {
     private final int storeCount;
     private final int storeFileCount;
     private final int storeRefCount;
-    private final int maxStoreFileRefCount;
+    private final int maxCompactedStoreFileRefCount;
     private final long compactingCellCount;
     private final long compactedCellCount;
     private final Size storeFileSize;
@@ -280,7 +280,7 @@ public final class RegionMetricsBuilder {
         int storeCount,
         int storeFileCount,
         int storeRefCount,
-        int maxStoreFileRefCount,
+        int maxCompactedStoreFileRefCount,
         final long compactingCellCount,
         long compactedCellCount,
         Size storeFileSize,
@@ -301,7 +301,7 @@ public final class RegionMetricsBuilder {
       this.storeCount = storeCount;
       this.storeFileCount = storeFileCount;
       this.storeRefCount = storeRefCount;
-      this.maxStoreFileRefCount = maxStoreFileRefCount;
+      this.maxCompactedStoreFileRefCount = maxCompactedStoreFileRefCount;
       this.compactingCellCount = compactingCellCount;
       this.compactedCellCount = compactedCellCount;
       this.storeFileSize = Preconditions.checkNotNull(storeFileSize);
@@ -341,8 +341,8 @@ public final class RegionMetricsBuilder {
     }
 
     @Override
-    public int getMaxStoreFileRefCount() {
+    public int getMaxCompactedStoreFileRefCount() {
-      return maxStoreFileRefCount;
+      return maxCompactedStoreFileRefCount;
     }
 
     @Override
@@ -433,8 +433,8 @@ public final class RegionMetricsBuilder {
           this.getStoreFileCount());
       Strings.appendKeyValue(sb, "storeRefCount",
         this.getStoreRefCount());
-      Strings.appendKeyValue(sb, "maxStoreFileRefCount",
+      Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
-        this.getMaxStoreFileRefCount());
+        this.getMaxCompactedStoreFileRefCount());
       Strings.appendKeyValue(sb, "uncompressedStoreFileSize",
           this.getUncompressedStoreFileSize());
       Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",

hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java

@@ -360,7 +360,7 @@ public final class ServerMetricsBuilder {
       int storeCount = 0;
       int storeFileCount = 0;
       int storeRefCount = 0;
-      int maxStoreFileRefCount = 0;
+      int maxCompactedStoreFileRefCount = 0;
       long uncompressedStoreFileSizeMB = 0;
       long storeFileSizeMB = 0;
       long memStoreSizeMB = 0;
@@ -376,8 +376,9 @@ public final class ServerMetricsBuilder {
         storeCount += r.getStoreCount();
         storeFileCount += r.getStoreFileCount();
         storeRefCount += r.getStoreRefCount();
-        int currentMaxStoreFileRefCount = r.getMaxStoreFileRefCount();
+        int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount();
-        maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount);
+        maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
+          currentMaxCompactedStoreFileRefCount);
         uncompressedStoreFileSizeMB += r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
         storeFileSizeMB += r.getStoreFileSize().get(Size.Unit.MEGABYTE);
         memStoreSizeMB += r.getMemStoreSize().get(Size.Unit.MEGABYTE);
@@ -399,7 +400,8 @@ public final class ServerMetricsBuilder {
       Strings.appendKeyValue(sb, "numberOfStores", storeCount);
       Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount);
       Strings.appendKeyValue(sb, "storeRefCount", storeRefCount);
-      Strings.appendKeyValue(sb, "maxStoreFileRefCount", maxStoreFileRefCount);
+      Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
+        maxCompactedStoreFileRefCount);
       Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB);
       Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB);
       if (uncompressedStoreFileSizeMB != 0) {

hbase-common/src/main/resources/hbase-default.xml

@@ -1891,19 +1891,25 @@ possible configurations would overwhelm and obscure the important.
     <name>hbase.regions.recovery.store.file.ref.count</name>
     <value>-1</value>
     <description>
-      Very large ref count on a file indicates
+      Very large number of ref count on a compacted
-      that it is a ref leak on that object. Such files
+      store file indicates that it is a ref leak
-      can not be removed even after it is invalidated
+      on that object(compacted store file).
-      via compaction. Only way to recover in such
+      Such files can not be removed after
-      scenario is to reopen the region which can
+      it is invalidated via compaction.
-      release all resources, like the refcount, leases, etc.
+      Only way to recover in such scenario is to
-      This config represents Store files Ref Count threshold
+      reopen the region which can release
-      value considered for reopening regions.
+      all resources, like the refcount,
-      Any region with store files ref count > this value
+      leases, etc. This config represents Store files Ref
-      would be eligible for reopening by master.
+      Count threshold value considered for reopening
-      Default value -1 indicates this feature is turned off.
+      regions. Any region with compacted store files
-      Only positive integer value should be provided to enable
+      ref count > this value would be eligible for
-      this feature.
+      reopening by master. Here, we get the max
+      refCount among all refCounts on all
+      compacted away store files that belong to a
+      particular region. Default value -1 indicates
+      this feature is turned off. Only positive
+      integer value should be provided to
+      enable this feature.
     </description>
   </property>
 </configuration>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -233,7 +233,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String STOREFILE_COUNT_DESC = "Number of Store Files";
   String STORE_REF_COUNT = "storeRefCount";
   String STORE_REF_COUNT_DESC = "Store reference count";
-  String MAX_STORE_FILE_REF_COUNT = "maxStoreFileRefCount";
+  String MAX_COMPACTED_STORE_FILE_REF_COUNT = "maxCompactedStoreFileRefCount";
   String MEMSTORE_SIZE = "memStoreSize";
   String MEMSTORE_SIZE_DESC = "Size of the memstore";
   String STOREFILE_SIZE = "storeFileSize";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java

@@ -160,7 +160,7 @@ public interface MetricsRegionWrapper {
 
   /**
    * @return the max number of references active on any store file among
-   *   all store files that belong to this region
+   *   all compacted store files that belong to this region
    */
-  long getMaxStoreFileRefCount();
+  long getMaxCompactedStoreFileRefCount();
 }

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java

@@ -218,9 +218,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
               MetricsRegionServerSource.STORE_REF_COUNT),
           this.regionWrapper.getStoreRefCount());
       mrb.addGauge(Interns.info(
-        regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT,
+        regionNamePrefix + MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT,
-        MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT),
+        MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT),
-        this.regionWrapper.getMaxStoreFileRefCount());
+        this.regionWrapper.getMaxCompactedStoreFileRefCount()
+      );
       mrb.addGauge(Interns.info(
               regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
               MetricsRegionServerSource.MEMSTORE_SIZE_DESC),

hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java

@@ -100,7 +100,7 @@ public class TestMetricsRegionSourceImpl {
     }
 
     @Override
-    public long getMaxStoreFileRefCount() {
+    public long getMaxCompactedStoreFileRefCount() {
       return 0;
     }
 

hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto

@@ -151,10 +151,10 @@ message RegionLoad {
   optional int32 store_ref_count = 21 [default = 0];
 
   /**
-   *  The max number of references active on single store file among all store files
+   *  The max number of references active on single store file among all compacted store files
    *  that belong to given region
    */
-  optional int32 max_store_file_ref_count = 22 [default = 0];
+  optional int32 max_compacted_store_file_ref_count = 22 [default = 0];
 }
 
 message UserLoad {

hbase-protocol/src/main/protobuf/ClusterStatus.proto

@@ -147,10 +147,10 @@ message RegionLoad {
   optional int32 store_ref_count = 21 [default = 0];
 
   /**
-   *  The max number of references active on single store file among all store files
+   *  The max number of references active on single store file among all compacted store files
    *  that belong to given region
    */
-  optional int32 max_store_file_ref_count = 22 [default = 0];
+  optional int32 max_compacted_store_file_ref_count = 22 [default = 0];
 }
 
 message UserLoad {

hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java

@@ -130,17 +130,18 @@ public class RegionsRecoveryChore extends ScheduledChore {
     for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
       Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
       for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
-        // For each region, each store file can have different ref counts
+        // For each region, each compacted store file can have different ref counts
-        // We need to find maximum of all such ref counts and if that max count
+        // We need to find maximum of all such ref counts and if that max count of compacted
-        // is beyond a threshold value, we should reopen the region.
+        // store files is beyond a threshold value, we should reopen the region.
-        // Here, we take max ref count of all store files and not the cumulative
+        // Here, we take max ref count of all compacted store files and not the cumulative
-        // count of all store files
+        // count of all compacted store files
-        final int maxStoreFileRefCount = regionMetrics.getMaxStoreFileRefCount();
+        final int maxCompactedStoreFileRefCount = regionMetrics
+          .getMaxCompactedStoreFileRefCount();
 
-        if (maxStoreFileRefCount > storeFileRefCountThreshold) {
+        if (maxCompactedStoreFileRefCount > storeFileRefCountThreshold) {
           final byte[] regionName = regionMetrics.getRegionName();
           prepareTableToReopenRegionsMap(tableToReopenRegionsMap, regionName,
-            maxStoreFileRefCount);
+            maxCompactedStoreFileRefCount);
         }
       }
     }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java

@@ -1660,7 +1660,7 @@ public class HRegionServer extends HasThread implements
     int stores = 0;
     int storefiles = 0;
     int storeRefCount = 0;
-    int maxStoreFileRefCount = 0;
+    int maxCompactedStoreFileRefCount = 0;
     int storeUncompressedSizeMB = 0;
     int storefileSizeMB = 0;
     int memstoreSizeMB = (int) (r.getMemStoreDataSize() / 1024 / 1024);
@@ -1676,8 +1676,9 @@ public class HRegionServer extends HasThread implements
       storefiles += store.getStorefilesCount();
       int currentStoreRefCount = store.getStoreRefCount();
       storeRefCount += currentStoreRefCount;
-      int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
+      int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
-      maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount);
+      maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
+        currentMaxCompactedStoreFileRefCount);
       storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024);
       storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
       //TODO: storefileIndexSizeKB is same with rootLevelIndexSizeKB?
@@ -1706,7 +1707,7 @@ public class HRegionServer extends HasThread implements
       .setStores(stores)
       .setStorefiles(storefiles)
       .setStoreRefCount(storeRefCount)
-      .setMaxStoreFileRefCount(maxStoreFileRefCount)
+      .setMaxCompactedStoreFileRefCount(maxCompactedStoreFileRefCount)
       .setStoreUncompressedSizeMB(storeUncompressedSizeMB)
       .setStorefileSizeMB(storefileSizeMB)
       .setMemStoreSizeMB(memstoreSizeMB)

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java

@@ -2779,18 +2779,19 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
   }
 
   /**
-   * @return get maximum ref count of storeFile among all HStore Files
+   * @return get maximum ref count of storeFile among all compacted HStore Files
    *   for the HStore
    */
-  public int getMaxStoreFileRefCount() {
+  public int getMaxCompactedStoreFileRefCount() {
-    OptionalInt maxStoreFileRefCount = this.storeEngine.getStoreFileManager()
+    OptionalInt maxCompactedStoreFileRefCount = this.storeEngine.getStoreFileManager()
-      .getStorefiles()
+      .getCompactedfiles()
       .stream()
       .filter(sf -> sf.getReader() != null)
       .filter(HStoreFile::isHFile)
       .mapToInt(HStoreFile::getRefCount)
       .max();
-    return maxStoreFileRefCount.isPresent() ? maxStoreFileRefCount.getAsInt() : 0;
+    return maxCompactedStoreFileRefCount.isPresent()
+      ? maxCompactedStoreFileRefCount.getAsInt() : 0;
   }
 
   void reportArchivedFilesForQuota(List<? extends StoreFile> archivedFiles, List<Long> fileSizes) {

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java

@@ -49,7 +49,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
   private Runnable runnable;
   private long numStoreFiles;
   private long storeRefCount;
-  private long maxStoreFileRefCount;
+  private long maxCompactedStoreFileRefCount;
   private long memstoreSize;
   private long storeFileSize;
   private long maxStoreFileAge;
@@ -127,8 +127,8 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
   }
 
   @Override
-  public long getMaxStoreFileRefCount() {
+  public long getMaxCompactedStoreFileRefCount() {
-    return maxStoreFileRefCount;
+    return maxCompactedStoreFileRefCount;
   }
 
   @Override
@@ -234,7 +234,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
     public void run() {
       long tempNumStoreFiles = 0;
       int tempStoreRefCount = 0;
-      int tempMaxStoreFileRefCount = 0;
+      int tempMaxCompactedStoreFileRefCount = 0;
       long tempMemstoreSize = 0;
       long tempStoreFileSize = 0;
       long tempMaxStoreFileAge = 0;
@@ -249,9 +249,9 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
           tempNumStoreFiles += store.getStorefilesCount();
           int currentStoreRefCount = store.getStoreRefCount();
           tempStoreRefCount += currentStoreRefCount;
-          int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
+          int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
-          tempMaxStoreFileRefCount = Math.max(tempMaxStoreFileRefCount,
+          tempMaxCompactedStoreFileRefCount = Math.max(tempMaxCompactedStoreFileRefCount,
-            currentMaxStoreFileRefCount);
+            currentMaxCompactedStoreFileRefCount);
           tempMemstoreSize += store.getMemStoreSize().getDataSize();
           tempStoreFileSize += store.getStorefilesSize();
           OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@@ -279,7 +279,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
 
       numStoreFiles = tempNumStoreFiles;
       storeRefCount = tempStoreRefCount;
-      maxStoreFileRefCount = tempMaxStoreFileRefCount;
+      maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount;
       memstoreSize = tempMemstoreSize;
       storeFileSize = tempStoreFileSize;
       maxStoreFileAge = tempMaxStoreFileAge;

hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java

@@ -214,7 +214,7 @@ public class TestRegionsRecoveryChore {
     Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(),
       Mockito.anyLong(), Mockito.anyLong());
 
-    // default maxStoreFileRefCount is -1 (no regions to be reopened using AM)
+    // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM)
     Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
     Mockito.verify(assignmentManager, Mockito.times(0))
       .getRegionInfo(Mockito.any());
@@ -380,7 +380,7 @@ public class TestRegionsRecoveryChore {
     return serverMetrics;
   }
 
-  private static RegionMetrics getRegionMetrics(byte[] regionName, int storeRefCount) {
+  private static RegionMetrics getRegionMetrics(byte[] regionName, int compactedStoreRefCount) {
     RegionMetrics regionMetrics = new RegionMetrics() {
 
       @Override
@@ -480,12 +480,12 @@ public class TestRegionsRecoveryChore {
 
       @Override
       public int getStoreRefCount() {
-        return storeRefCount;
+        return compactedStoreRefCount;
       }
 
       @Override
-      public int getMaxStoreFileRefCount() {
+      public int getMaxCompactedStoreFileRefCount() {
-        return storeRefCount;
+        return compactedStoreRefCount;
       }
 
     };

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java

@@ -66,7 +66,7 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
   }
 
   @Override
-  public long getMaxStoreFileRefCount() {
+  public long getMaxCompactedStoreFileRefCount() {
     return 0;
   }
 

src/main/asciidoc/_chapters/hbase-default.adoc

@@ -2047,19 +2047,25 @@ A comma-separated list of
 +
 .Description
 
-      Very large ref count on a file indicates
+      Very large number of ref count on a compacted
-      that it is a ref leak on that object. Such files
+      store file indicates that it is a ref leak
-      can not be removed even after it is invalidated
+      on that object(compacted store file).
-      via compaction. Only way to recover in such
+      Such files can not be removed after
-      scenario is to reopen the region which can
+      it is invalidated via compaction.
-      release all resources, like the refcount, leases, etc.
+      Only way to recover in such scenario is to
-      This config represents Store files Ref Count threshold
+      reopen the region which can release
-      value considered for reopening regions.
+      all resources, like the refcount,
-      Any region with store files ref count > this value
+      leases, etc. This config represents Store files Ref
-      would be eligible for reopening by master.
+      Count threshold value considered for reopening
-      Default value -1 indicates this feature is turned off.
+      regions. Any region with compacted store files
-      Only positive integer value should be provided to enable
+      ref count > this value would be eligible for
-      this feature.
+      reopening by master. Here, we get the max
+      refCount among all refCounts on all
+      compacted away store files that belong to a
+      particular region. Default value -1 indicates
+      this feature is turned off. Only positive
+      integer value should be provided to
+      enable this feature.
 
 +
 .Default