HBASE-24994 Add hedgedReadOpsInCurThread metric (#2365)

Signed-off-by: Duo Zhang <zhangduo@apache.org>

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java

@@ -486,6 +486,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
   String HEDGED_READ_WINS = "hedgedReadWins";
   String HEDGED_READ_WINS_DESC =
       "The number of times we started a hedged read and a hedged read won";
+  String HEDGED_READ_IN_CUR_THREAD = "hedgedReadOpsInCurThread";
+  String HEDGED_READ_IN_CUR_THREAD_DESC =
+    "The number of times we execute a hedged read in current thread as a fallback for task rejection";
 
   String TOTAL_BYTES_READ = "totalBytesRead";
   String TOTAL_BYTES_READ_DESC = "The total number of bytes read from HDFS";

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java

@@ -458,6 +458,8 @@ public class MetricsRegionServerSourceImpl
               .addCounter(Interns.info(HEDGED_READS, HEDGED_READS_DESC), rsWrap.getHedgedReadOps())
               .addCounter(Interns.info(HEDGED_READ_WINS, HEDGED_READ_WINS_DESC),
                       rsWrap.getHedgedReadWins())
+              .addCounter(Interns.info(HEDGED_READ_IN_CUR_THREAD, HEDGED_READ_IN_CUR_THREAD_DESC),
+                      rsWrap.getHedgedReadOpsInCurThread())
               .addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC),
                       rsWrap.getBlockedRequestsCount())
               .tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC),

hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java

@@ -452,6 +452,11 @@ public interface MetricsRegionServerWrapper {
    */
   long getHedgedReadWins();
 
+  /**
+   * @return Count of times a hedged read executes in current thread
+   */
+  long getHedgedReadOpsInCurThread();
+
   /**
    * @return Number of total bytes read from HDFS.
    */

hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java

@@ -928,6 +928,11 @@ class MetricsRegionServerWrapperImpl
     return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadWins();
   }
 
+  @Override
+  public long getHedgedReadOpsInCurThread() {
+    return this.dfsHedgedReadMetrics == null ? 0 : this.dfsHedgedReadMetrics.getHedgedReadOpsInCurThread();
+  }
+
   @Override
   public long getTotalBytesRead() {
     return FSDataInputStreamWrapper.getTotalBytesRead();

hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java

@@ -420,6 +420,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
     return 10;
   }
 
+  @Override
+  public long getHedgedReadOpsInCurThread() {
+    return 5;
+  }
+
   @Override
   public long getTotalBytesRead() {
     return 0;

src/main/asciidoc/_chapters/performance.adoc

@@ -801,6 +801,8 @@ See <<hbase_metrics>>  for more information.
   This could indicate that read requests are often slow, or that hedged reads are triggered too quickly.
 * hedgeReadOpsWin - the number of times the hedged read thread was faster than the original thread.
   This could indicate that a given RegionServer is having trouble servicing requests.
+* hedgedReadOpsInCurThread - the number of times hedged read was rejected from executor and needed to fallback to be executed in current thread.
+  This could indicate that current hedged read thread pool size is not appropriate.
 
 
 [[perf.deleting]]