HBASE-24994 Add hedgedReadOpsInCurThread metric (#2368)

Signed-off-by: Duo Zhang <zhangduo@apache.org>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -509,6 +509,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String HEDGED_READ_WINS = "hedgedReadWins";
   String HEDGED_READ_WINS_DESC =
     "The number of times we started a hedged read and a hedged read won";
+  String HEDGED_READ_IN_CUR_THREAD = "hedgedReadOpsInCurThread";
+  String HEDGED_READ_IN_CUR_THREAD_DESC =
+    "The number of times we execute a hedged read in current thread as a fallback for task rejection";
 
   String RPC_GET_REQUEST_COUNT = "rpcGetRequestCount";
   String RPC_GET_REQUEST_COUNT_DESC = "Number of rpc get requests this region server has answered.";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -358,6 +358,11 @@ public interface MetricsRegionServerWrapper {
    */
   long getHedgedReadWins();
 
+  /**
+   * @return Count of times a hedged read executes in current thread
+   */
+  long getHedgedReadOpsInCurThread();
+
   /**
    * @return Number of total bytes read from HDFS.
    */

hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -508,6 +508,8 @@ public class MetricsRegionServerSourceImpl
           .addCounter(Interns.info(HEDGED_READS, HEDGED_READS_DESC), rsWrap.getHedgedReadOps())
           .addCounter(Interns.info(HEDGED_READ_WINS, HEDGED_READ_WINS_DESC),
               rsWrap.getHedgedReadWins())
+          .addCounter(Interns.info(HEDGED_READ_IN_CUR_THREAD, HEDGED_READ_IN_CUR_THREAD_DESC),
+              rsWrap.getHedgedReadOpsInCurThread())
 
           .addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC),
             rsWrap.getBlockedRequestsCount())

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -842,6 +842,11 @@ class MetricsRegionServerWrapperImpl
     return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadWins();
   }
 
+  @Override
+  public long getHedgedReadOpsInCurThread() {
+    return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadOpsInCurThread();
+  }
+
   @Override
   public long getBlockedRequestsCount() {
     return blockedRequestsCount;

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -370,6 +370,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
     return 10;
   }
 
+  @Override
+  public long getHedgedReadOpsInCurThread() {
+    return 5;
+  }
+
   @Override
   public long getTotalBytesRead() {
     return 0;

src/main/asciidoc/_chapters/performance.adoc

@@ -760,6 +760,8 @@ See <<hbase_metrics>>  for more information.
   This could indicate that read requests are often slow, or that hedged reads are triggered too quickly.
 * hedgeReadOpsWin - the number of times the hedged read thread was faster than the original thread.
   This could indicate that a given RegionServer is having trouble servicing requests.
+* hedgedReadOpsInCurThread - the number of times hedged read was rejected from executor and needed to fallback to be executed in current thread.
+  This could indicate that current hedged read thread pool size is not appropriate.
 
 [[perf.deleting]]
 == Deleting from HBase