Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HubSpot Backport: HBASE-27188 Report maxStoreFileCount in jmx (#4609) 

Signed-off-by: Andrew Purtell <apurtell@apache.org>
This commit is contained in:
Bryan Beaudreault 2022-07-11 22:11:58 -04:00
parent 91c7ede9df
commit 9a0d069af7
11 changed files with 55 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
View File

@ -594,7 +594,9 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
"Number of rpc multi requests this RegionServer has answered.";
String RPC_MUTATE_REQUEST_COUNT = "rpcMutateRequestCount";
String RPC_MUTATE_REQUEST_COUNT_DESC =
"Number of rpc mutation requests this RegionServer has answered.";
"Number of rpc mutation requests this RegionServer has answered.";
String MAX_STOREFILE_COUNT = "maxStoreFileCount";
String MAX_STOREFILE_COUNT_DESC = "Max store file count across all regions";
String AVERAGE_REGION_SIZE = "averageRegionSize";
String AVERAGE_REGION_SIZE_DESC =
"Average region size over the RegionServer including memstore and storefile sizes.";

5
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
View File

@ -90,6 +90,11 @@ public interface MetricsRegionServerWrapper {
*/
long getNumStoreFiles();
/**
* Get the max number of store files across all regions of this region server.
*/
long getMaxStoreFiles();
/**
* Get the size of the memstore on this region server.
*/

5
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregate.java
View File

@ -84,6 +84,11 @@ public interface MetricsTableWrapperAggregate {
*/
long getNumStoreFiles(String table);
/**
* Get the max number of store files across all regions of this table
*/
long getMaxStoreFiles(String table);
/**
* @return Max age of store files for this table
*/

2
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
View File

@ -492,6 +492,8 @@ public class MetricsRegionServerSourceImpl
.addGauge(Interns.info(WALFILE_COUNT, WALFILE_COUNT_DESC), rsWrap.getNumWALFiles())
.addGauge(Interns.info(WALFILE_SIZE, WALFILE_SIZE_DESC), rsWrap.getWALFileSize())
.addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles())
.addGauge(Interns.info(MAX_STOREFILE_COUNT, MAX_STOREFILE_COUNT_DESC),
rsWrap.getMaxStoreFiles())
.addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemStoreSize())
.addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
.addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),

9
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableSourceImpl.java
View File

@ -290,8 +290,13 @@ public class MetricsTableSourceImpl implements MetricsTableSource {
tableWrapperAgg.getMemStoreSize(tableName.getNameAsString()));
mrb.addGauge(Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT,
MetricsRegionServerSource.STOREFILE_COUNT_DESC),
tableWrapperAgg.getNumStoreFiles(tableName.getNameAsString()));
mrb.addGauge(Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
tableWrapperAgg.getNumStoreFiles(tableName.getNameAsString()));
mrb.addGauge(
Interns.info(tableNamePrefix + MetricsRegionServerSource.MAX_STOREFILE_COUNT,
MetricsRegionServerSource.MAX_STOREFILE_COUNT_DESC),
tableWrapperAgg.getMaxStoreFiles(tableName.getNameAsString()));
mrb.addGauge(
Interns.info(tableNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
MetricsRegionServerSource.STOREFILE_SIZE_DESC),
tableWrapperAgg.getStoreFileSize(tableName.getNameAsString()));
mrb.addGauge(Interns.info(tableNamePrefix + MetricsTableSource.TABLE_SIZE,

5
hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperStub.java
View File

@ -83,6 +83,11 @@ public class MetricsTableWrapperStub implements MetricsTableWrapperAggregate {
return 33;
}
@Override
public long getMaxStoreFiles(String table) {
return 8;
}
@Override
public long getMaxStoreFileAge(String table) {
return 44;

10
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
View File

@ -78,6 +78,7 @@ class MetricsRegionServerWrapperImpl
private volatile long numStoreFiles = 0;
private volatile long memstoreSize = 0;
private volatile long storeFileSize = 0;
private volatile long maxStoreFileCount = 0;
private volatile long maxStoreFileAge = 0;
private volatile long minStoreFileAge = 0;
private volatile long avgStoreFileAge = 0;
@ -465,6 +466,11 @@ class MetricsRegionServerWrapperImpl
return numStoreFiles;
}
@Override
public long getMaxStoreFiles() {
return maxStoreFileCount;
}
@Override
public long getMaxStoreFileAge() {
return maxStoreFileAge;
@ -726,6 +732,7 @@ class MetricsRegionServerWrapperImpl
long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
long tempMaxStoreFileCount = 0;
long avgAgeNumerator = 0, numHFiles = 0;
long tempMinStoreFileAge = Long.MAX_VALUE;
long tempReadRequestsCount = 0, tempFilteredReadRequestsCount = 0,
@ -805,6 +812,8 @@ class MetricsRegionServerWrapperImpl
tempMemstoreSize += store.getMemStoreSize().getDataSize();
tempStoreFileSize += store.getStorefilesSize();
tempMaxStoreFileCount = Math.max(tempMaxStoreFileCount, store.getStorefilesCount());
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
if (storeMaxStoreFileAge.isPresent() &&
storeMaxStoreFileAge.getAsLong() > tempMaxStoreFileAge) {
@ -898,6 +907,7 @@ class MetricsRegionServerWrapperImpl
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileCount = tempMaxStoreFileCount;
maxStoreFileAge = tempMaxStoreFileAge;
if (regionCount > 0) {
averageRegionSize = (memstoreSize + storeFileSize) / regionCount;

11
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsTableWrapperAggregateImpl.java
View File

@ -77,6 +77,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr
familyName = store.getColumnFamilyName();
mt.storeFileCount += store.getStorefilesCount();
mt.maxStoreFileCount = Math.max(mt.maxStoreFileCount, store.getStorefilesCount());
mt.memstoreSize += (store.getMemStoreSize().getDataSize()
+ store.getMemStoreSize().getHeapSize() + store.getMemStoreSize().getOffHeapSize());
mt.storeFileSize += store.getStorefilesSize();
@ -265,6 +266,15 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr
return metricsTable.storeFileCount;
}
@Override
public long getMaxStoreFiles(String table) {
MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
if (metricsTable == null) {
return 0;
}
return metricsTable.maxStoreFileCount;
}
@Override
public long getMaxStoreFileAge(String table) {
MetricsTableValues metricsTable = metricsTableMap.get(TableName.valueOf(table));
@ -336,6 +346,7 @@ public class MetricsTableWrapperAggregateImpl implements MetricsTableWrapperAggr
long regionCount;
long storeCount;
long storeFileCount;
long maxStoreFileCount;
long storeFileSize;
long maxStoreFileAge;
long minStoreFileAge = Long.MAX_VALUE;

5
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
View File

@ -60,6 +60,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 300;
}
@Override
public long getMaxStoreFiles() {
return 23;
}
@Override
public long getMemStoreSize() {
return 1025;

1
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
View File

@ -74,6 +74,7 @@ public class TestMetricsRegionServer {
HELPER.assertGauge("regionServerStartTime", 100, serverSource);
HELPER.assertGauge("regionCount", 101, serverSource);
HELPER.assertGauge("storeCount", 2, serverSource);
HELPER.assertGauge("maxStoreFileCount", 23, serverSource);
HELPER.assertGauge("maxStoreFileAge", 2, serverSource);
HELPER.assertGauge("minStoreFileAge", 2, serverSource);
HELPER.assertGauge("avgStoreFileAge", 2, serverSource);

1
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsTableAggregate.java
View File

@ -90,6 +90,7 @@ public class TestMetricsTableAggregate {
HELPER.assertGauge(pre + "regionCount", 11, agg);
HELPER.assertGauge(pre + "storeCount", 22, agg);
HELPER.assertGauge(pre + "storeFileCount", 33, agg);
HELPER.assertGauge(pre + "maxStoreFileCount", 8, agg);
HELPER.assertGauge(pre + "maxStoreFileAge", 44, agg);
HELPER.assertGauge(pre + "minStoreFileAge", 55, agg);
HELPER.assertGauge(pre + "avgStoreFileAge", 66, agg);