diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 7ac2501607f..b27696c1dfc 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -252,6 +252,10 @@ public interface MetricsRegionServerSource extends BaseSource { String HEDGED_READ_WINS_DESC = "The number of times we started a hedged read and a hedged read won"; + String BLOCKED_REQUESTS_COUNT = "blockedRequestCount"; + String BLOCKED_REQUESTS_COUNT_DESC = "The number of blocked requests because of memstore size is " + + "larger than blockingMemStoreSize"; + String SPLIT_KEY = "splitTime"; String FLUSH_KEY = "flushTime"; } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index e0b5e1b0d24..0f62dc68188 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -256,4 +256,9 @@ public interface MetricsRegionServerWrapper { * @return Count of times a hedged read beat out the primary read. */ public long getHedgedReadWins(); + + /** + * @return Count of requests blocked because the memstore size is larger than blockingMemStoreSize + */ + public long getBlockedRequestsCount(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index d3414e20cb8..4cd83382499 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -237,6 +237,9 @@ public class MetricsRegionServerSourceImpl .addCounter(Interns.info(HEDGED_READ_WINS, HEDGED_READ_WINS_DESC), rsWrap.getHedgedReadWins()) + .addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC), + rsWrap.getBlockedRequestsCount()) + .tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC), rsWrap.getZookeeperQuorum()) .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), rsWrap.getServerName()) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index d6a07314ffb..1615a2548d9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -287,6 +287,16 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver { // final Counter readRequestsCount = new Counter(); final Counter writeRequestsCount = new Counter(); + // Number of requests blocked by memstore size. + private final Counter blockedRequestsCount = new Counter(); + + /** + * @return the number of blocked requests count. + */ + public long getBlockedRequestsCount() { + return this.blockedRequestsCount.get(); + } + // Compaction counters final AtomicLong compactionsFinished = new AtomicLong(0L); final AtomicLong compactionNumFilesCompacted = new AtomicLong(0L); @@ -3121,6 +3131,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver { // if (this.getRegionInfo().isMetaRegion()) return; if (this.memstoreSize.get() > this.blockingMemStoreSize) { + blockedRequestsCount.increment(); requestFlush(); throw new RegionTooBusyException("Above memstore limit, " + "regionName=" + (this.getRegionInfo() == null ? "unknown" : @@ -5769,7 +5780,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver { // public static final long FIXED_OVERHEAD = ClassSize.align( ClassSize.OBJECT + ClassSize.ARRAY + - 41 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT + + 42 * ClassSize.REFERENCE + 2 * Bytes.SIZEOF_INT + (12 * Bytes.SIZEOF_LONG) + 4 * Bytes.SIZEOF_BOOLEAN); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 52eafb94fd3..d8ad6fe5d89 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -76,6 +76,7 @@ class MetricsRegionServerWrapperImpl private volatile long flushedCellsSize = 0; private volatile long compactedCellsSize = 0; private volatile long majorCompactedCellsSize = 0; + private volatile long blockedRequestsCount = 0L; private CacheStats cacheStats; private ScheduledExecutorService executor; @@ -441,6 +442,7 @@ class MetricsRegionServerWrapperImpl long tempFlushedCellsSize = 0; long tempCompactedCellsSize = 0; long tempMajorCompactedCellsSize = 0; + long tempBlockedRequestsCount = 0L; for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { tempNumMutationsWithoutWAL += r.numMutationsWithoutWAL.get(); @@ -449,6 +451,7 @@ class MetricsRegionServerWrapperImpl tempWriteRequestsCount += r.writeRequestsCount.get(); tempCheckAndMutateChecksFailed += r.checkAndMutateChecksFailed.get(); tempCheckAndMutateChecksPassed += r.checkAndMutateChecksPassed.get(); + tempBlockedRequestsCount += r.getBlockedRequestsCount(); tempNumStores += r.stores.size(); for (Store store : r.stores.values()) { tempNumStoreFiles += store.getStorefilesCount(); @@ -515,6 +518,7 @@ class MetricsRegionServerWrapperImpl flushedCellsSize = tempFlushedCellsSize; compactedCellsSize = tempCompactedCellsSize; majorCompactedCellsSize = tempMajorCompactedCellsSize; + blockedRequestsCount = tempBlockedRequestsCount; } } @@ -527,4 +531,9 @@ class MetricsRegionServerWrapperImpl public long getHedgedReadWins() { return this.dfsHedgedReadMetrics == null? 0: this.dfsHedgedReadMetrics.getHedgedReadWins(); } + + @Override + public long getBlockedRequestsCount() { + return blockedRequestsCount; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 58c61c6559d..e936789350e 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -250,4 +250,9 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe public long getHedgedReadWins() { return 10; } + + @Override + public long getBlockedRequestsCount() { + return 0; + } }