HBASE-14178 regionserver blocks because of waiting for offsetLock

Signed-off-by: zhangduo <zhangduo@apache.org>

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java

@@ -431,6 +431,48 @@ public class CacheConfig {
     return isBlockCacheEnabled() && this.prefetchOnOpen;
   }
 
+  /**
+   * Return true if we may find this type of block in block cache.
+   * <p/>
+   * TODO: today {@code family.isBlockCacheEnabled()} only means {@code cacheDataOnRead}, so here we
+   * consider lots of other configurations such as {@code cacheDataOnWrite}. We should fix this in
+   * the future, {@code cacheDataOnWrite} should honor the CF level {@code isBlockCacheEnabled}
+   * configuration.
+   */
+  public boolean shouldReadBlockFromCache(BlockType blockType) {
+    if (!isBlockCacheEnabled()) {
+      return false;
+    }
+    if (cacheDataOnRead) {
+      return true;
+    }
+    if (prefetchOnOpen) {
+      return true;
+    }
+    if (cacheDataOnWrite) {
+      return true;
+    }
+    if (blockType == null) {
+      return true;
+    }
+    if (blockType.getCategory() == BlockCategory.BLOOM ||
+            blockType.getCategory() == BlockCategory.INDEX) {
+      return true;
+    }
+    return false;
+  }
+
+  /**
+   * If we make sure the block could not be cached, we will not acquire the lock
+   * otherwise we will acquire lock
+   */
+  public boolean shouldLockOnCacheMiss(BlockType blockType) {
+    if (blockType == null) {
+      return true;
+    }
+    return shouldCacheBlockOnRead(blockType.getCategory());
+  }
+
   @Override
   public String toString() {
     if (!isBlockCacheEnabled()) {

hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java

@@ -393,12 +393,11 @@ public class HFileReaderV2 extends AbstractHFileReader {
     TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock");
     try {
       while (true) {
+        // Check cache for block. If found return.
+        if (cacheConf.shouldReadBlockFromCache(expectedBlockType)) {
           if (useLock) {
             lockEntry = offsetLock.getLockEntry(dataBlockOffset);
           }
-
-        // Check cache for block. If found return.
-        if (cacheConf.isBlockCacheEnabled()) {
           // Try and get the block from the block cache. If the useLock variable is true then this
           // is the second time through the loop and it should not be counted as a block cache miss.
           HFileBlock cachedBlock = getCachedBlock(cacheKey, cacheBlock, useLock, isCompaction,
@@ -423,13 +422,14 @@ public class HFileReaderV2 extends AbstractHFileReader {
             // Cache-hit. Return!
             return cachedBlock;
           }
-          // Carry on, please load.
+          if (!useLock && cacheBlock && cacheConf.shouldLockOnCacheMiss(expectedBlockType)) {
-        }
-        if (!useLock) {
             // check cache again with lock
             useLock = true;
             continue;
           }
+          // Carry on, please load.
+        }
+
         if (Trace.isTracing()) {
           traceScope.getSpan().addTimelineAnnotation("blockCacheMiss");
         }