HBASE-27890 Expose a getter on Connection/AsyncConnection for getting public access to connection metrics

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnection.java

@@ -237,4 +237,9 @@ public interface AsyncConnection extends Closeable {
    */
   @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.HBCK)
   Hbck getHbck(ServerName masterServer) throws IOException;
+
+  /** Returns a statistical sample of {@link MetricsConnection} */
+  default MetricsConnectionSnapshot getMetrics() {
+    return null;
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncConnectionImpl.java

@@ -453,4 +453,9 @@ public class AsyncConnectionImpl implements AsyncConnection {
   Optional<MetricsConnection> getConnectionMetrics() {
     return metrics;
   }
+
+  @Override
+  public MetricsConnectionSnapshot getMetrics() {
+    return metrics.map(MetricsConnection::snapshot).orElse(null);
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/Connection.java

@@ -216,4 +216,9 @@ public interface Connection extends Abortable, Closeable {
   default Hbck getHbck(ServerName masterServer) throws IOException {
     return toAsyncConnection().getHbck(masterServer);
   }
+
+  /** Returns a statistical sample of {@link MetricsConnection} */
+  default MetricsConnectionSnapshot getMetrics() {
+    return null;
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionOverAsyncConnection.java

@@ -226,4 +226,9 @@ class ConnectionOverAsyncConnection implements Connection {
   public String toString() {
     return "connection-over-async-connection-0x" + Integer.toHexString(hashCode());
   }
+
+  @Override
+  public MetricsConnectionSnapshot getMetrics() {
+    return conn.getMetrics();
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnection.java

@@ -18,6 +18,7 @@
 package org.apache.hadoop.hbase.client;
 
 import static com.codahale.metrics.MetricRegistry.name;
+import static org.apache.hadoop.hbase.client.MetricsConnectionSnapshot.snapshotMap;
 import static org.apache.hadoop.hbase.util.ConcurrentMapUtils.computeIfAbsent;
 
 import com.codahale.metrics.Counter;
@@ -36,6 +37,9 @@ import java.util.concurrent.TimeUnit;
 import java.util.function.Supplier;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.client.MetricsConnectionSnapshot.CallTrackerSnapshot;
+import org.apache.hadoop.hbase.client.MetricsConnectionSnapshot.RegionStatsSnapshot;
+import org.apache.hadoop.hbase.client.MetricsConnectionSnapshot.RunnerStatsSnapshot;
 import org.apache.hadoop.hbase.util.Bytes;
 import org.apache.hadoop.ipc.RemoteException;
 import org.apache.yetus.audience.InterfaceAudience;
@@ -219,6 +223,10 @@ public final class MetricsConnection implements StatisticTrackable {
       this.respHist.update(stats.getResponseSizeBytes());
     }
 
+    String getName() {
+      return name;
+    }
+
     @Override
     public String toString() {
       return "CallTracker:" + name;
@@ -247,12 +255,12 @@ public final class MetricsConnection implements StatisticTrackable {
   protected static class RunnerStats {
     final Counter normalRunners;
     final Counter delayRunners;
-    final Histogram delayIntevalHist;
+    final Histogram delayIntervalHist;
 
     public RunnerStats(MetricRegistry registry) {
       this.normalRunners = registry.counter(name(MetricsConnection.class, "normalRunnersCount"));
       this.delayRunners = registry.counter(name(MetricsConnection.class, "delayRunnersCount"));
-      this.delayIntevalHist =
+      this.delayIntervalHist =
         registry.histogram(name(MetricsConnection.class, "delayIntervalHist"));
     }
 
@@ -265,11 +273,11 @@ public final class MetricsConnection implements StatisticTrackable {
     }
 
     public void updateDelayInterval(long interval) {
-      this.delayIntevalHist.update(interval);
+      this.delayIntervalHist.update(interval);
     }
   }
 
-  private ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>> serverStats =
+  private final ConcurrentHashMap<ServerName, ConcurrentMap<byte[], RegionStats>> serverStats =
     new ConcurrentHashMap<>();
 
   public void updateServerStats(ServerName serverName, byte[] regionName, Object r) {
@@ -359,6 +367,8 @@ public final class MetricsConnection implements StatisticTrackable {
   private final Counter nsLookups;
   private final Counter nsLookupsFailed;
   private final Timer overloadedBackoffTimer;
+  private final RatioGauge executorPoolUsageRatio;
+  private final RatioGauge metaPoolUsageRatio;
 
   // dynamic metrics
 
@@ -379,46 +389,20 @@ public final class MetricsConnection implements StatisticTrackable {
     this.scope = scope;
     addThreadPools(batchPool, metaPool);
     this.registry = new MetricRegistry();
-    this.registry.register(getExecutorPoolName(), new RatioGauge() {
+    this.executorPoolUsageRatio = new RatioGauge() {
       @Override
       protected Ratio getRatio() {
-        int numerator = 0;
-        int denominator = 0;
-        for (Supplier<ThreadPoolExecutor> poolSupplier : batchPools) {
-          ThreadPoolExecutor pool = poolSupplier.get();
-          if (pool != null) {
-            int activeCount = pool.getActiveCount();
-            int maxPoolSize = pool.getMaximumPoolSize();
-            /* The max thread usage ratio among batch pools of all connections */
-            if (numerator == 0 || (numerator * maxPoolSize) < (activeCount * denominator)) {
-              numerator = activeCount;
-              denominator = maxPoolSize;
+        return calculateThreadPoolUsageRatio(batchPools);
       }
-          }
-        }
-        return Ratio.of(numerator, denominator);
-      }
-    });
-    this.registry.register(getMetaPoolName(), new RatioGauge() {
+    };
+    this.metaPoolUsageRatio = new RatioGauge() {
       @Override
       protected Ratio getRatio() {
-        int numerator = 0;
-        int denominator = 0;
-        for (Supplier<ThreadPoolExecutor> poolSupplier : metaPools) {
-          ThreadPoolExecutor pool = poolSupplier.get();
-          if (pool != null) {
-            int activeCount = pool.getActiveCount();
-            int maxPoolSize = pool.getMaximumPoolSize();
-            /* The max thread usage ratio among meta lookup pools of all connections */
-            if (numerator == 0 || (numerator * maxPoolSize) < (activeCount * denominator)) {
-              numerator = activeCount;
-              denominator = maxPoolSize;
+        return calculateThreadPoolUsageRatio(metaPools);
       }
-          }
-        }
-        return Ratio.of(numerator, denominator);
-      }
-    });
+    };
+    this.registry.register(getExecutorPoolName(), executorPoolUsageRatio);
+    this.registry.register(getMetaPoolName(), metaPoolUsageRatio);
     this.connectionCount = registry.counter(name(this.getClass(), "connectionCount", scope));
     this.metaCacheHits = registry.counter(name(this.getClass(), "metaCacheHits", scope));
     this.metaCacheMisses = registry.counter(name(this.getClass(), "metaCacheMisses", scope));
@@ -745,4 +729,51 @@ public final class MetricsConnection implements StatisticTrackable {
   public void incrNsLookupsFailed() {
     this.nsLookupsFailed.inc();
   }
+
+  private RatioGauge.Ratio
+    calculateThreadPoolUsageRatio(Iterable<Supplier<ThreadPoolExecutor>> pools) {
+    int numerator = 0;
+    int denominator = 0;
+    for (Supplier<ThreadPoolExecutor> poolSupplier : pools) {
+      ThreadPoolExecutor pool = poolSupplier.get();
+      if (pool != null) {
+        int activeCount = pool.getActiveCount();
+        int maxPoolSize = pool.getMaximumPoolSize();
+        /* The max thread usage ratio among pools of all connections */
+        if (numerator == 0 || (numerator * maxPoolSize) < (activeCount * denominator)) {
+          numerator = activeCount;
+          denominator = maxPoolSize;
+        }
+      }
+    }
+    return RatioGauge.Ratio.of(numerator, denominator);
+  }
+
+  synchronized MetricsConnectionSnapshot snapshot() {
+    return MetricsConnectionSnapshot.newBuilder().connectionCount(connectionCount.getCount())
+      .metaCacheHits(metaCacheHits.getCount()).metaCacheMisses(metaCacheMisses.getCount())
+      .getTrackerSnap(CallTrackerSnapshot.snapshot(getTracker))
+      .scanTrackerSnap(CallTrackerSnapshot.snapshot(scanTracker))
+      .appendTrackerSnap(CallTrackerSnapshot.snapshot(appendTracker))
+      .deleteTrackerSnap(CallTrackerSnapshot.snapshot(deleteTracker))
+      .incrementTrackerSnap(CallTrackerSnapshot.snapshot(incrementTracker))
+      .putTrackerSnap(CallTrackerSnapshot.snapshot(putTracker))
+      .multiTrackerSnap(CallTrackerSnapshot.snapshot(multiTracker))
+      .runnerStatsSnap(RunnerStatsSnapshot.snapshot(runnerStats))
+      .metaCacheNumClearServer(metaCacheNumClearServer.getCount())
+      .metaCacheNumClearRegion(metaCacheNumClearRegion.getCount())
+      .hedgedReadOps(hedgedReadOps.getCount()).hedgedReadWin(hedgedReadWin.getCount())
+      .concurrentCallsPerServerHistSnap(concurrentCallsPerServerHist.getSnapshot())
+      .numActionsPerServerHistSnap(numActionsPerServerHist.getSnapshot())
+      .nsLookups(nsLookups.getCount()).nsLookupsFailed(nsLookupsFailed.getCount())
+      .overloadedBackoffTimerSnap(overloadedBackoffTimer.getSnapshot())
+      .executorPoolUsageRatio(executorPoolUsageRatio.getValue())
+      .metaPoolUsageRatio(metaPoolUsageRatio.getValue())
+      .rpcTimersSnap(snapshotMap(rpcTimers, Timer::getSnapshot))
+      .rpcHistSnap(snapshotMap(rpcHistograms, Histogram::getSnapshot))
+      .cacheDroppingExceptions(snapshotMap(cacheDroppingExceptions, Counter::getCount))
+      .rpcCounters(snapshotMap(rpcCounters, Counter::getCount)).serverStats(snapshotMap(serverStats,
+        regionStatsMap -> snapshotMap(regionStatsMap, RegionStatsSnapshot::snapshot)))
+      .build();
+  }
 }

hbase-client/src/main/java/org/apache/hadoop/hbase/client/MetricsConnectionSnapshot.java

@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.client;
+
+import com.codahale.metrics.Snapshot;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.function.Function;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.yetus.audience.InterfaceAudience;
+
+/**
+ * A statistical sample of {@link MetricsConnection}.
+ */
+@InterfaceAudience.Public
+public final class MetricsConnectionSnapshot {
+  private final long connectionCount;
+  private final long metaCacheHits;
+  private final long metaCacheMisses;
+  private final CallTrackerSnapshot getTrackerSnap;
+  private final CallTrackerSnapshot scanTrackerSnap;
+  private final CallTrackerSnapshot appendTrackerSnap;
+  private final CallTrackerSnapshot deleteTrackerSnap;
+  private final CallTrackerSnapshot incrementTrackerSnap;
+  private final CallTrackerSnapshot putTrackerSnap;
+  private final CallTrackerSnapshot multiTrackerSnap;
+  private final RunnerStatsSnapshot runnerStatsSnap;
+  private final long metaCacheNumClearServer;
+  private final long metaCacheNumClearRegion;
+  private final long hedgedReadOps;
+  private final long hedgedReadWin;
+  private final Snapshot concurrentCallsPerServerHistSnap;
+  private final Snapshot numActionsPerServerHistSnap;
+  private final long nsLookups;
+  private final long nsLookupsFailed;
+  private final Snapshot overloadedBackoffTimerSnap;
+  private final double executorPoolUsageRatio;
+  private final double metaPoolUsageRatio;
+  private final Map<String, Snapshot> rpcTimersSnap;
+  private final Map<String, Snapshot> rpcHistSnap;
+  private final Map<String, Long> cacheDroppingExceptions;
+  private final Map<String, Long> rpcCounters;
+  private final Map<ServerName, Map<byte[], RegionStatsSnapshot>> serverStats;
+
+  public static class CallTrackerSnapshot {
+    private final String name;
+    private final Snapshot callTimerSnap;
+    private final Snapshot reqHistSnap;
+    private final Snapshot respHistSnap;
+
+    public CallTrackerSnapshot(String name, Snapshot callTimerSnap, Snapshot reqHistSnap,
+      Snapshot respHistSnap) {
+      this.name = name;
+      this.callTimerSnap = callTimerSnap;
+      this.reqHistSnap = reqHistSnap;
+      this.respHistSnap = respHistSnap;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public Snapshot getCallTimerSnap() {
+      return callTimerSnap;
+    }
+
+    public Snapshot getReqHistSnap() {
+      return reqHistSnap;
+    }
+
+    public Snapshot getRespHistSnap() {
+      return respHistSnap;
+    }
+
+    static CallTrackerSnapshot snapshot(MetricsConnection.CallTracker callTracker) {
+      return new CallTrackerSnapshot(callTracker.getName(), callTracker.callTimer.getSnapshot(),
+        callTracker.reqHist.getSnapshot(), callTracker.respHist.getSnapshot());
+    }
+  }
+
+  public static class RunnerStatsSnapshot {
+    private final long normalRunners;
+    private final long delayRunners;
+    private final Snapshot delayIntervalHistSnap;
+
+    public RunnerStatsSnapshot(long normalRunners, long delayRunners,
+      Snapshot delayIntervalHistSnap) {
+      this.normalRunners = normalRunners;
+      this.delayRunners = delayRunners;
+      this.delayIntervalHistSnap = delayIntervalHistSnap;
+    }
+
+    public long getNormalRunners() {
+      return normalRunners;
+    }
+
+    public long getDelayRunners() {
+      return delayRunners;
+    }
+
+    public Snapshot getDelayIntervalHistSnap() {
+      return delayIntervalHistSnap;
+    }
+
+    static RunnerStatsSnapshot snapshot(MetricsConnection.RunnerStats runnerStats) {
+      return new RunnerStatsSnapshot(runnerStats.normalRunners.getCount(),
+        runnerStats.delayRunners.getCount(), runnerStats.delayIntervalHist.getSnapshot());
+    }
+  }
+
+  public static class RegionStatsSnapshot {
+    private final String name;
+    private final Snapshot memstoreLoadHistSnap;
+    private final Snapshot heapOccupancyHistSnap;
+
+    public RegionStatsSnapshot(String name, Snapshot memstoreLoadHistSnap,
+      Snapshot heapOccupancyHistSnap) {
+      this.name = name;
+      this.memstoreLoadHistSnap = memstoreLoadHistSnap;
+      this.heapOccupancyHistSnap = heapOccupancyHistSnap;
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    public Snapshot getMemstoreLoadHistSnap() {
+      return memstoreLoadHistSnap;
+    }
+
+    public Snapshot getHeapOccupancyHistSnap() {
+      return heapOccupancyHistSnap;
+    }
+
+    static RegionStatsSnapshot snapshot(MetricsConnection.RegionStats regionStats) {
+      return new RegionStatsSnapshot(regionStats.name, regionStats.memstoreLoadHist.getSnapshot(),
+        regionStats.heapOccupancyHist.getSnapshot());
+    }
+  }
+
+  public MetricsConnectionSnapshot(long connectionCount, long metaCacheHits, long metaCacheMisses,
+    CallTrackerSnapshot getTrackerSnap, CallTrackerSnapshot scanTrackerSnap,
+    CallTrackerSnapshot appendTrackerSnap, CallTrackerSnapshot deleteTrackerSnap,
+    CallTrackerSnapshot incrementTrackerSnap, CallTrackerSnapshot putTrackerSnap,
+    CallTrackerSnapshot multiTrackerSnap, RunnerStatsSnapshot runnerStatsSnap,
+    long metaCacheNumClearServer, long metaCacheNumClearRegion, long hedgedReadOps,
+    long hedgedReadWin, Snapshot concurrentCallsPerServerHistSnap,
+    Snapshot numActionsPerServerHistSnap, long nsLookups, long nsLookupsFailed,
+    Snapshot overloadedBackoffTimerSnap, double executorPoolUsageRatio, double metaPoolUsageRatio,
+    Map<String, Snapshot> rpcTimersSnap, Map<String, Snapshot> rpcHistSnap,
+    Map<String, Long> cacheDroppingExceptions, Map<String, Long> rpcCounters,
+    Map<ServerName, Map<byte[], RegionStatsSnapshot>> serverStats) {
+    this.connectionCount = connectionCount;
+    this.metaCacheHits = metaCacheHits;
+    this.metaCacheMisses = metaCacheMisses;
+    this.getTrackerSnap = getTrackerSnap;
+    this.scanTrackerSnap = scanTrackerSnap;
+    this.appendTrackerSnap = appendTrackerSnap;
+    this.deleteTrackerSnap = deleteTrackerSnap;
+    this.incrementTrackerSnap = incrementTrackerSnap;
+    this.putTrackerSnap = putTrackerSnap;
+    this.multiTrackerSnap = multiTrackerSnap;
+    this.runnerStatsSnap = runnerStatsSnap;
+    this.metaCacheNumClearServer = metaCacheNumClearServer;
+    this.metaCacheNumClearRegion = metaCacheNumClearRegion;
+    this.hedgedReadOps = hedgedReadOps;
+    this.hedgedReadWin = hedgedReadWin;
+    this.concurrentCallsPerServerHistSnap = concurrentCallsPerServerHistSnap;
+    this.numActionsPerServerHistSnap = numActionsPerServerHistSnap;
+    this.nsLookups = nsLookups;
+    this.nsLookupsFailed = nsLookupsFailed;
+    this.overloadedBackoffTimerSnap = overloadedBackoffTimerSnap;
+    this.executorPoolUsageRatio = executorPoolUsageRatio;
+    this.metaPoolUsageRatio = metaPoolUsageRatio;
+    this.rpcTimersSnap = rpcTimersSnap;
+    this.rpcHistSnap = rpcHistSnap;
+    this.cacheDroppingExceptions = cacheDroppingExceptions;
+    this.rpcCounters = rpcCounters;
+    this.serverStats = serverStats;
+  }
+
+  static MetricsConnectionSnapshotBuilder newBuilder() {
+    return new MetricsConnectionSnapshotBuilder();
+  }
+
+  static class MetricsConnectionSnapshotBuilder {
+    private long connectionCount;
+    private long metaCacheHits;
+    private long metaCacheMisses;
+    private CallTrackerSnapshot getTrackerSnap;
+    private CallTrackerSnapshot scanTrackerSnap;
+    private CallTrackerSnapshot appendTrackerSnap;
+    private CallTrackerSnapshot deleteTrackerSnap;
+    private CallTrackerSnapshot incrementTrackerSnap;
+    private CallTrackerSnapshot putTrackerSnap;
+    private CallTrackerSnapshot multiTrackerSnap;
+    private RunnerStatsSnapshot runnerStatsSnap;
+    private long metaCacheNumClearServer;
+    private long metaCacheNumClearRegion;
+    private long hedgedReadOps;
+    private long hedgedReadWin;
+    private Snapshot concurrentCallsPerServerHistSnap;
+    private Snapshot numActionsPerServerHistSnap;
+    private long nsLookups;
+    private long nsLookupsFailed;
+    private Snapshot overloadedBackoffTimerSnap;
+    private double executorPoolUsageRatio;
+    private double metaPoolUsageRatio;
+    private Map<String, Snapshot> rpcTimersSnap;
+    private Map<String, Snapshot> rpcHistSnap;
+    private Map<String, Long> cacheDroppingExceptions;
+    private Map<String, Long> rpcCounters;
+    private Map<ServerName, Map<byte[], RegionStatsSnapshot>> serverStats;
+
+    public MetricsConnectionSnapshotBuilder connectionCount(long connectionCount) {
+      this.connectionCount = connectionCount;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder metaCacheHits(long metaCacheHits) {
+      this.metaCacheHits = metaCacheHits;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder metaCacheMisses(long metaCacheMisses) {
+      this.metaCacheMisses = metaCacheMisses;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder getTrackerSnap(CallTrackerSnapshot getTrackerSnap) {
+      this.getTrackerSnap = getTrackerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder scanTrackerSnap(CallTrackerSnapshot scanTrackerSnap) {
+      this.scanTrackerSnap = scanTrackerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      appendTrackerSnap(CallTrackerSnapshot appendTrackerSnap) {
+      this.appendTrackerSnap = appendTrackerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      deleteTrackerSnap(CallTrackerSnapshot deleteTrackerSnap) {
+      this.deleteTrackerSnap = deleteTrackerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      incrementTrackerSnap(CallTrackerSnapshot incrementTrackerSnap) {
+      this.incrementTrackerSnap = incrementTrackerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder putTrackerSnap(CallTrackerSnapshot putTrackerSnap) {
+      this.putTrackerSnap = putTrackerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder multiTrackerSnap(CallTrackerSnapshot multiTrackerSnap) {
+      this.multiTrackerSnap = multiTrackerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder runnerStatsSnap(RunnerStatsSnapshot runnerStatsSnap) {
+      this.runnerStatsSnap = runnerStatsSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder metaCacheNumClearServer(long metaCacheNumClearServer) {
+      this.metaCacheNumClearServer = metaCacheNumClearServer;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder metaCacheNumClearRegion(long metaCacheNumClearRegion) {
+      this.metaCacheNumClearRegion = metaCacheNumClearRegion;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder hedgedReadOps(long hedgedReadOps) {
+      this.hedgedReadOps = hedgedReadOps;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder hedgedReadWin(long hedgedReadWin) {
+      this.hedgedReadWin = hedgedReadWin;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      concurrentCallsPerServerHistSnap(Snapshot concurrentCallsPerServerHistSnap) {
+      this.concurrentCallsPerServerHistSnap = concurrentCallsPerServerHistSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      numActionsPerServerHistSnap(Snapshot numActionsPerServerHistSnap) {
+      this.numActionsPerServerHistSnap = numActionsPerServerHistSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder nsLookups(long nsLookups) {
+      this.nsLookups = nsLookups;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder nsLookupsFailed(long nsLookupsFailed) {
+      this.nsLookupsFailed = nsLookupsFailed;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      overloadedBackoffTimerSnap(Snapshot overloadedBackoffTimerSnap) {
+      this.overloadedBackoffTimerSnap = overloadedBackoffTimerSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder executorPoolUsageRatio(double executorPoolUsageRatio) {
+      this.executorPoolUsageRatio = executorPoolUsageRatio;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder metaPoolUsageRatio(double metaPoolUsageRatio) {
+      this.metaPoolUsageRatio = metaPoolUsageRatio;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder rpcTimersSnap(Map<String, Snapshot> rpcTimersSnap) {
+      this.rpcTimersSnap = rpcTimersSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder rpcHistSnap(Map<String, Snapshot> rpcHistSnap) {
+      this.rpcHistSnap = rpcHistSnap;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      cacheDroppingExceptions(Map<String, Long> cacheDroppingExceptions) {
+      this.cacheDroppingExceptions = cacheDroppingExceptions;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder rpcCounters(Map<String, Long> rpcCounters) {
+      this.rpcCounters = rpcCounters;
+      return this;
+    }
+
+    public MetricsConnectionSnapshotBuilder
+      serverStats(Map<ServerName, Map<byte[], RegionStatsSnapshot>> serverStats) {
+      this.serverStats = serverStats;
+      return this;
+    }
+
+    public MetricsConnectionSnapshot build() {
+      return new MetricsConnectionSnapshot(connectionCount, metaCacheHits, metaCacheMisses,
+        getTrackerSnap, scanTrackerSnap, appendTrackerSnap, deleteTrackerSnap, incrementTrackerSnap,
+        putTrackerSnap, multiTrackerSnap, runnerStatsSnap, metaCacheNumClearServer,
+        metaCacheNumClearRegion, hedgedReadOps, hedgedReadWin, concurrentCallsPerServerHistSnap,
+        numActionsPerServerHistSnap, nsLookups, nsLookupsFailed, overloadedBackoffTimerSnap,
+        executorPoolUsageRatio, metaPoolUsageRatio, rpcTimersSnap, rpcHistSnap,
+        cacheDroppingExceptions, rpcCounters, serverStats);
+    }
+  }
+
+  public long getConnectionCount() {
+    return connectionCount;
+  }
+
+  public long getMetaCacheHits() {
+    return metaCacheHits;
+  }
+
+  public long getMetaCacheMisses() {
+    return metaCacheMisses;
+  }
+
+  public CallTrackerSnapshot getGetTrackerSnap() {
+    return getTrackerSnap;
+  }
+
+  public CallTrackerSnapshot getScanTrackerSnap() {
+    return scanTrackerSnap;
+  }
+
+  public CallTrackerSnapshot getAppendTrackerSnap() {
+    return appendTrackerSnap;
+  }
+
+  public CallTrackerSnapshot getDeleteTrackerSnap() {
+    return deleteTrackerSnap;
+  }
+
+  public CallTrackerSnapshot getIncrementTrackerSnap() {
+    return incrementTrackerSnap;
+  }
+
+  public CallTrackerSnapshot getPutTrackerSnap() {
+    return putTrackerSnap;
+  }
+
+  public CallTrackerSnapshot getMultiTrackerSnap() {
+    return multiTrackerSnap;
+  }
+
+  public RunnerStatsSnapshot getRunnerStatsSnap() {
+    return runnerStatsSnap;
+  }
+
+  public long getMetaCacheNumClearServer() {
+    return metaCacheNumClearServer;
+  }
+
+  public long getMetaCacheNumClearRegion() {
+    return metaCacheNumClearRegion;
+  }
+
+  public long getHedgedReadOps() {
+    return hedgedReadOps;
+  }
+
+  public long getHedgedReadWin() {
+    return hedgedReadWin;
+  }
+
+  public Snapshot getConcurrentCallsPerServerHistSnap() {
+    return concurrentCallsPerServerHistSnap;
+  }
+
+  public Snapshot getNumActionsPerServerHistSnap() {
+    return numActionsPerServerHistSnap;
+  }
+
+  public long getNsLookups() {
+    return nsLookups;
+  }
+
+  public long getNsLookupsFailed() {
+    return nsLookupsFailed;
+  }
+
+  public Snapshot getOverloadedBackoffTimerSnap() {
+    return overloadedBackoffTimerSnap;
+  }
+
+  public double getExecutorPoolUsageRatio() {
+    return executorPoolUsageRatio;
+  }
+
+  public double getMetaPoolUsageRatio() {
+    return metaPoolUsageRatio;
+  }
+
+  public Map<String, Snapshot> getRpcTimersSnap() {
+    return rpcTimersSnap;
+  }
+
+  public Map<String, Snapshot> getRpcHistSnap() {
+    return rpcHistSnap;
+  }
+
+  public Map<String, Long> getCacheDroppingExceptions() {
+    return cacheDroppingExceptions;
+  }
+
+  public Map<String, Long> getRpcCounters() {
+    return rpcCounters;
+  }
+
+  public Map<ServerName, Map<byte[], RegionStatsSnapshot>> getServerStats() {
+    return serverStats;
+  }
+
+  static <K, IN, OUT> Map<K, OUT> snapshotMap(Map<K, IN> map, Function<IN, OUT> snapshotFunction) {
+    Map<K, OUT> snapshot = new HashMap<>();
+    for (Map.Entry<K, IN> entry : map.entrySet()) {
+      snapshot.put(entry.getKey(), snapshotFunction.apply(entry.getValue()));
+    }
+    return snapshot;
+  }
+}

hbase-client/src/test/java/org/apache/hadoop/hbase/client/TestMetricsConnection.java

@@ -99,7 +99,7 @@ public class TestMetricsConnection {
   }
 
   @Test
-  public void testMetricsWithMutiConnections() throws IOException {
+  public void testMetricsWithMultiConnections() throws IOException {
     Configuration conf = new Configuration();
     conf.setBoolean(MetricsConnection.CLIENT_SIDE_METRICS_ENABLED_KEY, true);
     conf.set(MetricsConnection.METRICS_SCOPE_KEY, "unit-test");
@@ -181,6 +181,7 @@ public class TestMetricsConnection {
         MetricsConnection.newCallStats(), null);
     }
 
+    MetricsConnectionSnapshot metricsConnSnap = METRICS.snapshot();
     final String rpcCountPrefix = "rpcCount_" + ClientService.getDescriptor().getName() + "_";
     final String rpcFailureCountPrefix =
       "rpcFailureCount_" + ClientService.getDescriptor().getName() + "_";
@@ -192,6 +193,7 @@ public class TestMetricsConnection {
       metricKey = rpcCountPrefix + method;
       metricVal = METRICS.getRpcCounters().get(metricKey).getCount();
       assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal >= loop);
+      assertEquals(metricVal, metricsConnSnap.getRpcCounters().get(metricKey).longValue());
 
       metricKey = rpcFailureCountPrefix + method;
       counter = METRICS.getRpcCounters().get(metricKey);
@@ -199,9 +201,12 @@ public class TestMetricsConnection {
       if (method.equals("Get") || method.equals("Mutate")) {
         // no failure
         assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == 0);
+        assertEquals(0, metricsConnSnap.getRpcCounters().getOrDefault(metricKey, 0L).longValue());
       } else {
         // has failure
         assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop);
+        assertEquals(loop,
+          metricsConnSnap.getRpcCounters().getOrDefault(metricKey, 0L).longValue());
       }
     }
 
@@ -210,18 +215,22 @@ public class TestMetricsConnection {
     counter = METRICS.getRpcCounters().get(metricKey);
     metricVal = (counter != null) ? counter.getCount() : 0;
     assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop);
+    assertEquals(loop, metricsConnSnap.getRpcCounters().getOrDefault(metricKey, 0L).longValue());
 
     // local exception
     metricKey = "rpcLocalExceptions_CallTimeoutException";
     counter = METRICS.getRpcCounters().get(metricKey);
     metricVal = (counter != null) ? counter.getCount() : 0;
     assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop);
+    assertEquals(loop, metricsConnSnap.getRpcCounters().getOrDefault(metricKey, 0L).longValue());
 
     // total exception
     metricKey = "rpcTotalExceptions";
     counter = METRICS.getRpcCounters().get(metricKey);
     metricVal = (counter != null) ? counter.getCount() : 0;
     assertTrue("metric: " + metricKey + " val: " + metricVal, metricVal == loop * 2);
+    assertEquals(loop * 2,
+      metricsConnSnap.getRpcCounters().getOrDefault(metricKey, 0L).longValue());
 
     for (MetricsConnection.CallTracker t : new MetricsConnection.CallTracker[] {
       METRICS.getGetTracker(), METRICS.getScanTracker(), METRICS.getMultiTracker(),

hbase-server/src/test/java/org/apache/hadoop/hbase/client/ClientPushbackTestBase.java

@@ -153,7 +153,7 @@ public abstract class ClientPushbackTestBase {
 
     assertEquals(1, runnerStats.delayRunners.getCount());
     assertEquals(1, runnerStats.normalRunners.getCount());
-    assertEquals("", runnerStats.delayIntevalHist.getSnapshot().getMean(), (double) backoffTime,
+    assertEquals("", runnerStats.delayIntervalHist.getSnapshot().getMean(), (double) backoffTime,
       0.1);
 
     latch.await(backoffTime * 2, TimeUnit.MILLISECONDS);