HBASE-11527 Cluster free memory limit check should consider L2 block cache size also when L2 cache is onheap. (Anoop)

hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java

@@ -122,6 +122,15 @@ public class HeapMemorySizeUtil {
     // L1 block cache is always on heap
     float l1CachePercent = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY,
         HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT);
+    float l2CachePercent = getL2BlockCacheHeapPercent(conf);
+    return l1CachePercent + l2CachePercent;
+  }
+
+  /**
+   * @param conf
+   * @return The on heap size for L2 block cache.
+   */
+  public static float getL2BlockCacheHeapPercent(Configuration conf) {
     float l2CachePercent = 0.0F;
     String bucketCacheIOEngineName = conf.get(HConstants.BUCKET_CACHE_IOENGINE_KEY, null);
     // L2 block cache can be on heap when IOEngine is "heap"
@@ -131,6 +140,6 @@ public class HeapMemorySizeUtil {
       l2CachePercent = bucketCachePercentage < 1 ? bucketCachePercentage
           : (bucketCachePercentage * 1024 * 1024) / mu.getMax();
     }
-    return l1CachePercent + l2CachePercent;
+    return l2CachePercent;
   }
 }

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java

@@ -68,6 +68,7 @@ public class HeapMemoryManager {
   private float blockCachePercent;
   private float blockCachePercentMinRange;
   private float blockCachePercentMaxRange;
+  private float l2BlockCachePercent;
 
   private final ResizableBlockCache blockCache;
   private final FlushRequester memStoreFlusher;
@@ -147,7 +148,8 @@ public class HeapMemoryManager {
     }
 
     int gml = (int) (globalMemStorePercentMaxRange * CONVERT_TO_PERCENTAGE);
-    int bcul = (int) (blockCachePercentMinRange * CONVERT_TO_PERCENTAGE);
+    this.l2BlockCachePercent = HeapMemorySizeUtil.getL2BlockCacheHeapPercent(conf);
+    int bcul = (int) ((blockCachePercentMinRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
     if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
       throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
           + "the threshold required for successful cluster operation. "
@@ -158,7 +160,7 @@ public class HeapMemoryManager {
           + blockCachePercentMinRange);
     }
     gml = (int) (globalMemStorePercentMinRange * CONVERT_TO_PERCENTAGE);
-    bcul = (int) (blockCachePercentMaxRange * CONVERT_TO_PERCENTAGE);
+    bcul = (int) ((blockCachePercentMaxRange + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
     if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
       throw new RuntimeException("Current heap configuration for MemStore and BlockCache exceeds "
           + "the threshold required for successful cluster operation. "
@@ -249,7 +251,7 @@ public class HeapMemoryManager {
           blockCacheSize = blockCachePercentMaxRange;
         }
         int gml = (int) (memstoreSize * CONVERT_TO_PERCENTAGE);
-        int bcul = (int) (blockCacheSize * CONVERT_TO_PERCENTAGE);
+        int bcul = (int) ((blockCacheSize + l2BlockCachePercent) * CONVERT_TO_PERCENTAGE);
         if (CONVERT_TO_PERCENTAGE - (gml + bcul) < CLUSTER_MINIMUM_MEMORY_THRESHOLD) {
           LOG.info("Current heap configuration from HeapMemoryTuner exceeds "
               + "the threshold required for successful cluster operation. "

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java

@@ -29,6 +29,7 @@ import java.util.Iterator;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.CoordinatedStateManager;
 import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HConstants;
 import org.apache.hadoop.hbase.Server;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.SmallTests;
@@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheStats;
 import org.apache.hadoop.hbase.io.hfile.Cacheable;
 import org.apache.hadoop.hbase.io.hfile.CachedBlock;
 import org.apache.hadoop.hbase.io.hfile.ResizableBlockCache;
+import org.apache.hadoop.hbase.io.util.HeapMemorySizeUtil;
 import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerContext;
 import org.apache.hadoop.hbase.regionserver.HeapMemoryManager.TunerResult;
 import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
@@ -246,6 +248,55 @@ public class TestHeapMemoryManager {
     assertEquals(oldBlockCacheSize, blockCache.maxSize);
   }
 
+  @Test
+  public void testWhenL2BlockCacheIsOnHeap() throws Exception {
+    HeapMemoryManager heapMemoryManager = null;
+    BlockCacheStub blockCache = new BlockCacheStub((long) (maxHeapSize * 0.4));
+    MemstoreFlusherStub memStoreFlusher = new MemstoreFlusherStub((long) (maxHeapSize * 0.3));
+    Configuration conf = HBaseConfiguration.create();
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.1f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.1f);
+
+    conf.setFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_KEY, 0.4F);
+    conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.3F);
+    conf.setFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0.1F);
+    conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "heap");
+
+    conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000);
+    conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class,
+        HeapMemoryTuner.class);
+
+    try {
+      heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(
+          conf));
+      fail("Should have failed as the collective heap memory need is above 80%");
+    } catch (Exception e) {
+    }
+
+    // Change the max/min ranges for memstore and bock cache so as to pass the criteria check
+    conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.6f);
+    conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.6f);
+    heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(
+        conf));
+    long oldMemstoreSize = memStoreFlusher.memstoreSize;
+    long oldBlockCacheSize = blockCache.maxSize;
+    heapMemoryManager.start();
+    CustomHeapMemoryTuner.memstoreSize = 0.4f;
+    CustomHeapMemoryTuner.blockCacheSize = 0.4f;
+    Thread.sleep(1500);
+    // The size should not get changes as the collection of memstore size and L1 and L2 block cache
+    // size will cross the ax allowed 80% mark
+    assertEquals(oldMemstoreSize, memStoreFlusher.memstoreSize);
+    assertEquals(oldBlockCacheSize, blockCache.maxSize);
+    CustomHeapMemoryTuner.memstoreSize = 0.1f;
+    CustomHeapMemoryTuner.blockCacheSize = 0.5f;
+    Thread.sleep(1500);
+    assertHeapSpace(0.1f, memStoreFlusher.memstoreSize);
+    assertHeapSpace(0.5f, blockCache.maxSize);
+  }
+
   private void assertHeapSpace(float expectedHeapPercentage, long currentHeapSpace) {
     long expected = (long) (this.maxHeapSize * expectedHeapPercentage);
     assertEquals(expected, currentHeapSpace);