From 550d253ead76aaa367fdd88fe48a9ac9bad96fb4 Mon Sep 17 00:00:00 2001 From: Enis Soztutar Date: Thu, 21 Apr 2016 13:33:26 -0700 Subject: [PATCH] HBASE-15671 Add per-table metrics on memstore, storefile and regionsize (Alicia Ying Shu) --- .../regionserver/MetricsTableSource.java | 6 ++ .../MetricsTableWrapperAggregate.java | 15 +++++ .../regionserver/MetricsTableSourceImpl.java | 9 +++ .../TestMetricsTableSourceImpl.java | 15 +++++ .../MetricsTableWrapperAggregateImpl.java | 62 ++++++++++++++++++- .../regionserver/MetricsTableWrapperStub.java | 15 +++++ .../TestMetricsTableAggregate.java | 12 ++-- 7 files changed, 129 insertions(+), 5 deletions(-) diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSource.java index 6daccfb26c9..7d1f5d00716 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSource.java @@ -30,6 +30,12 @@ public interface MetricsTableSource extends Comparable { String WRITE_REQUEST_COUNT_DESC = "Number fo write requests"; String TOTAL_REQUEST_COUNT = "totalRequestCount"; String TOTAL_REQUEST_COUNT_DESC = "Number fo total requests"; + String MEMSTORE_SIZE = "memstoreSize"; + String MEMSTORE_SIZE_DESC = "The size of memory stores"; + String STORE_FILE_SIZE = "storeFileSize"; + String STORE_FILE_SIZE_DESC = "The size of store files size"; + String TABLE_SIZE = "tableSize"; + String TABLE_SIZE_DESC = "Total size of the table in the region server"; String getTableName(); diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java index e87eb7e0abe..85ea4f6701d 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java @@ -39,4 +39,19 @@ public interface MetricsTableWrapperAggregate { * Get the total number of requests that have been issued against this table */ long getTotalRequestsCount(String table); + + /** + * Get the memory store size against this table + */ + long getMemstoresSize(String table); + + /** + * Get the store file size against this table + */ + long getStoreFilesSize(String table); + + /** + * Get the table region size against this table + */ + long getTableSize(String table); } diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java index 1e3734c3d2d..33a807f1706 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java @@ -131,6 +131,15 @@ public class MetricsTableSourceImpl implements MetricsTableSource { mrb.addCounter(Interns.info(tableNamePrefix + MetricsTableSource.TOTAL_REQUEST_COUNT, MetricsTableSource.TOTAL_REQUEST_COUNT_DESC), tableWrapperAgg.getTotalRequestsCount(tableName.getNameAsString())); + mrb.addGauge(Interns.info(tableNamePrefix + MetricsTableSource.MEMSTORE_SIZE, + MetricsTableSource.MEMSTORE_SIZE_DESC), + tableWrapperAgg.getMemstoresSize(tableName.getNameAsString())); + mrb.addGauge(Interns.info(tableNamePrefix + MetricsTableSource.STORE_FILE_SIZE, + MetricsTableSource.STORE_FILE_SIZE_DESC), + tableWrapperAgg.getStoreFilesSize(tableName.getNameAsString())); + mrb.addGauge(Interns.info(tableNamePrefix + MetricsTableSource.TABLE_SIZE, + MetricsTableSource.TABLE_SIZE_DESC), + tableWrapperAgg.getTableSize(tableName.getNameAsString())); } } } diff --git a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java index 5423ddbe9ca..2717817e0ac 100644 --- a/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java +++ b/hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableSourceImpl.java @@ -91,6 +91,21 @@ public class TestMetricsTableSourceImpl { return 30; } + @Override + public long getMemstoresSize(String table) { + return 1000; + } + + @Override + public long getStoreFilesSize(String table) { + return 2000; + } + + @Override + public long getTableSize(String table) { + return 3000; + } + public String getTableName() { return tableName; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java index 3e6552cd442..c5f0f7bef3f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.regionserver; import java.io.Closeable; import java.io.IOException; import java.util.HashMap; -import java.util.HashSet; import java.util.Map; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -69,6 +68,13 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr metricsTable = new MetricsTableValues(); localMetricsTableMap.put(tbl, metricsTable); } + long tempStorefilesSize = 0; + for (Store store : r.getStores()) { + tempStorefilesSize += store.getStorefilesSize(); + } + metricsTable.setMemstoresSize(metricsTable.getMemstoresSize() + r.getMemstoreSize()); + metricsTable.setStoreFilesSize(metricsTable.getStoreFilesSize() + tempStorefilesSize); + metricsTable.setTableSize(metricsTable.getMemstoresSize() + metricsTable.getStoreFilesSize()); metricsTable.setReadRequestsCount(metricsTable.getReadRequestsCount() + r.getReadRequestsCount()); metricsTable.setWriteRequestsCount(metricsTable.getWriteRequestsCount() + r.getWriteRequestsCount()); metricsTable.setTotalRequestsCount(metricsTable.getReadRequestsCount() + metricsTable.getWriteRequestsCount()); @@ -126,6 +132,33 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr return metricsTable.getTotalRequestsCount(); } + @Override + public long getMemstoresSize(String table) { + MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table)); + if (metricsTable == null) + return 0; + else + return metricsTable.getMemstoresSize(); + } + + @Override + public long getStoreFilesSize(String table) { + MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table)); + if (metricsTable == null) + return 0; + else + return metricsTable.getStoreFilesSize(); + } + + @Override + public long getTableSize(String table) { + MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table)); + if (metricsTable == null) + return 0; + else + return metricsTable.getTableSize(); + } + @Override public void close() throws IOException { tableMetricsUpdateTask.cancel(true); @@ -136,6 +169,9 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr private long totalRequestsCount; private long readRequestsCount; private long writeRequestsCount; + private long memstoresSize; + private long storeFilesSize; + private long tableSize; public long getTotalRequestsCount() { return totalRequestsCount; @@ -160,6 +196,30 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr public void setWriteRequestsCount(long writeRequestsCount) { this.writeRequestsCount = writeRequestsCount; } + + public long getMemstoresSize() { + return memstoresSize; + } + + public void setMemstoresSize(long memstoresSize) { + this.memstoresSize = memstoresSize; + } + + public long getStoreFilesSize() { + return storeFilesSize; + } + + public void setStoreFilesSize(long storeFilesSize) { + this.storeFilesSize = storeFilesSize; + } + + public long getTableSize() { + return tableSize; + } + + public void setTableSize(long tableSize) { + this.tableSize = tableSize; + } } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java index 9c7d9827b5e..6fd8dd7135b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java @@ -41,6 +41,21 @@ public class MetricsTableWrapperStub implements MetricsTableWrapperAggregate { return 30; } + @Override + public long getMemstoresSize(String table) { + return 1000; + } + + @Override + public long getStoreFilesSize(String table) { + return 2000; + } + + @Override + public long getTableSize(String table) { + return 3000; + } + public String getTableName() { return tableName; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java index afde53f6395..c1142981426 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java @@ -34,15 +34,19 @@ public class TestMetricsTableAggregate { @Test public void testTableWrapperAggregateMetrics() throws IOException { - String tableName = "testRequestCount"; + String tableName = "testTableMetrics"; MetricsTableWrapperStub tableWrapper = new MetricsTableWrapperStub(tableName); CompatibilitySingletonFactory.getInstance(MetricsRegionServerSourceFactory.class) .createTable(tableName, tableWrapper); MetricsTableAggregateSource agg = CompatibilitySingletonFactory .getInstance(MetricsRegionServerSourceFactory.class).getTableAggregate(); - HELPER.assertCounter("Namespace_default_table_testRequestCount_metric_readRequestCount", 10, agg); - HELPER.assertCounter("Namespace_default_table_testRequestCount_metric_writeRequestCount", 20, agg); - HELPER.assertCounter("Namespace_default_table_testRequestCount_metric_totalRequestCount", 30, agg); + HELPER.assertCounter("Namespace_default_table_testTableMetrics_metric_readRequestCount", 10, agg); + HELPER.assertCounter("Namespace_default_table_testTableMetrics_metric_writeRequestCount", 20, agg); + HELPER.assertCounter("Namespace_default_table_testTableMetrics_metric_totalRequestCount", 30, agg); + + HELPER.assertGauge("Namespace_default_table_testTableMetrics_metric_memstoreSize", 1000, agg); + HELPER.assertGauge("Namespace_default_table_testTableMetrics_metric_storeFileSize", 2000, agg); + HELPER.assertGauge("Namespace_default_table_testTableMetrics_metric_tableSize", 3000, agg); } } \ No newline at end of file