From d07230a7596bec8d9bb30f6faaf7f74ef2c87474 Mon Sep 17 00:00:00 2001 From: Enis Soztutar Date: Tue, 22 Mar 2016 14:46:27 -0700 Subject: [PATCH] HBASE-15412 Add average region size metric (Alicia Ying Shu) Conflicts: hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java --- .../MetricsRegionServerSource.java | 3 +++ .../MetricsRegionServerWrapper.java | 5 +++++ .../MetricsRegionServerSourceImpl.java | 2 +- .../MetricsRegionServerWrapperImpl.java | 12 +++++++++++ .../MetricsRegionServerWrapperStub.java | 5 +++++ .../regionserver/TestRegionServerMetrics.java | 21 +++++++++++++++++++ 6 files changed, 47 insertions(+), 1 deletion(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 4fd5728f4ba..cdd896776ac 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -422,4 +422,7 @@ public interface MetricsRegionServerSource extends BaseSource { String RPC_MUTATE_REQUEST_COUNT = "rpcMutateRequestCount"; String RPC_MUTATE_REQUEST_COUNT_DESC = "Number of rpc mutation requests this region server has answered."; + String AVERAGE_REGION_SIZE = "averageRegionSize"; + String AVERAGE_REGION_SIZE_DESC = + "Average region size over the region server including memstore and storefile sizes."; } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index bb782bb30aa..101ea3d59cb 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -322,4 +322,9 @@ public interface MetricsRegionServerWrapper { * Get the number of rpc mutate requests to this region server. */ long getRpcMutateRequestsCount(); + + /** + * Get the average region size to this region server. + */ + long getAverageRegionSize(); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index fd99045372e..f1c2b823373 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -328,6 +328,7 @@ public class MetricsRegionServerSourceImpl rsWrap.getNumReferenceFiles()) .addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC), rsWrap.getStartCode()) + .addGauge(Interns.info(AVERAGE_REGION_SIZE, AVERAGE_REGION_SIZE_DESC), rsWrap.getAverageRegionSize()) .addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC), rsWrap.getTotalRequestCount()) .addCounter(Interns.info(READ_REQUEST_COUNT, READ_REQUEST_COUNT_DESC), @@ -412,7 +413,6 @@ public class MetricsRegionServerSourceImpl rsWrap.getCompactedCellsSize()) .addCounter(Interns.info(MAJOR_COMPACTED_CELLS_SIZE, MAJOR_COMPACTED_CELLS_SIZE_DESC), rsWrap.getMajorCompactedCellsSize()) - .addCounter(Interns.info(BLOCKED_REQUESTS_COUNT, BLOCKED_REQUESTS_COUNT_DESC), rsWrap.getBlockedRequestsCount()) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index af4b13b1af6..3d9f1b80a8c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -81,6 +81,7 @@ class MetricsRegionServerWrapperImpl private volatile long compactedCellsSize = 0; private volatile long majorCompactedCellsSize = 0; private volatile long blockedRequestsCount = 0L; + private volatile long averageRegionSize = 0L; private CacheStats cacheStats; private ScheduledExecutorService executor; @@ -538,6 +539,7 @@ class MetricsRegionServerWrapperImpl long tempMajorCompactedCellsSize = 0; long tempBlockedRequestsCount = 0L; + int regionCount = 0; for (Region r : regionServer.getOnlineRegionsLocalContext()) { tempNumMutationsWithoutWAL += r.getNumMutationsWithoutWAL(); tempDataInMemoryWithoutWAL += r.getDataInMemoryWithoutWAL(); @@ -582,6 +584,7 @@ class MetricsRegionServerWrapperImpl if (r.getRegionInfo().getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { hdfsBlocksDistributionSecondaryRegions.add(distro); } + regionCount++; } float localityIndex = @@ -627,6 +630,9 @@ class MetricsRegionServerWrapperImpl memstoreSize = tempMemstoreSize; storeFileSize = tempStoreFileSize; maxStoreFileAge = tempMaxStoreFileAge; + if (regionCount > 0) { + averageRegionSize = (memstoreSize + storeFileSize) / regionCount; + } if (tempMinStoreFileAge != Long.MAX_VALUE) { minStoreFileAge = tempMinStoreFileAge; } @@ -664,4 +670,10 @@ class MetricsRegionServerWrapperImpl public long getBlockedRequestsCount() { return blockedRequestsCount; } + + @Override + public long getAverageRegionSize() { + return averageRegionSize; + } + } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 8965d094fb8..3f4aa734844 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -314,4 +314,9 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe public int getSplitQueueSize() { return 0; } + + @Override + public long getAverageRegionSize() { + return 10000000; + } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java index af657e3179e..d068217391f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java @@ -625,4 +625,25 @@ public class TestRegionServerMetrics { table.close(); } + + @Test + public void testAverageRegionSize() throws Exception { + TableName tableName = TableName.valueOf("testAverageRegionSize"); + byte[] cf = Bytes.toBytes("d"); + byte[] row = Bytes.toBytes("rk"); + byte[] qualifier = Bytes.toBytes("qual"); + byte[] val = Bytes.toBytes("Value"); + + //Force a hfile. + Table t = TEST_UTIL.createTable(tableName, cf); + Put p = new Put(row); + p.addColumn(cf, qualifier, val); + t.put(p); + TEST_UTIL.getHBaseAdmin().flush(tableName); + + metricsRegionServer.getRegionServerWrapper().forceRecompute(); + assertTrue(metricsHelper.getGaugeDouble("averageRegionSize", serverSource) > 0.0); + + t.close(); + } }