HBASE-14178 regionserver blocks because of waiting for offsetLock

Signed-off-by: zhangduo <zhangduo@apache.org>

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java

@@ -431,6 +431,48 @@ public class CacheConfig {
     return isBlockCacheEnabled() && this.prefetchOnOpen;
   }
 
+  /**
+   * Return true if we may find this type of block in block cache.
+   * <p>
+   * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we
+   * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in
+   * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled}
+   * configuration.
+   */
+  public boolean shouldReadBlockFromCache(BlockType blockType) {
+    if (!isBlockCacheEnabled()) {
+      return false;
+    }
+    if (cacheDataOnRead) {
+      return true;
+    }
+    if (prefetchOnOpen) {
+      return true;
+    }
+    if (cacheDataOnWrite) {
+      return true;
+    }
+    if (blockType == null) {
+      return true;
+    }
+    if (blockType.getCategory() == BlockCategory.BLOOM ||
+            blockType.getCategory() == BlockCategory.INDEX) {
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * If we make sure the block could not be cached, we will not acquire the lock
+   * otherwise we will acquire lock
+   */
+  public boolean shouldLockOnCacheMiss(BlockType blockType) {
+    if (blockType == null) {
+      return true;
+    }
+    return shouldCacheBlockOnRead(blockType.getCategory());
+  }
+
   @Override
   public String toString() {
     if (!isBlockCacheEnabled()) {

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java

@@ -1431,12 +1431,11 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
     TraceScope traceScope = Trace.startSpan("HFileReaderImpl.readBlock");
     try {
       while (true) {
-        if (useLock) {
-          lockEntry = offsetLock.getLockEntry(dataBlockOffset);
-        }
-
         // Check cache for block. If found return.
-        if (cacheConf.isBlockCacheEnabled()) {
+        if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) {
+          if (useLock) {
+            lockEntry = offsetLock.getLockEntry(dataBlockOffset);
+          }
           // Try and get the block from the block cache. If the useLock variable is true then this
           // is the second time through the loop and it should not be counted as a block cache miss.
           HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
@@ -1461,13 +1460,15 @@ public class HFileReaderImpl implements HFile.Reader, Configurable {
             // Cache-hit. Return!
             return cachedBlock;
           }
+
+          if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) {
+            // check cache again with lock
+            useLock = true;
+            continue;
+          }
           // Carry on, please load.
         }
-        if (!useLock) {
-          // check cache again with lock
-          useLock = true;
-          continue;
-        }
+
         if (Trace.isTracing()) {
           traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
         }