Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-15135 Add metrics for storefile age

This commit is contained in:
Mikhail Antonov 2016-02-22 02:16:40 -08:00
parent ed290cf829
commit e58c0385a7
17 changed files with 401 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
View File

@ -155,6 +155,14 @@ public interface MetricsRegionServerSource extends BaseSource {
String MEMSTORE_SIZE = "memStoreSize";
String MEMSTORE_SIZE_DESC = "Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";
String MAX_STORE_FILE_AGE = "maxStoreFileAge";
String MIN_STORE_FILE_AGE = "minStoreFileAge";
String AVG_STORE_FILE_AGE = "avgStoreFileAge";
String NUM_REFERENCE_FILES = "numReferenceFiles";
String MAX_STORE_FILE_AGE_DESC = "Max age of store files hosted on this region server";
String MIN_STORE_FILE_AGE_DESC = "Min age of store files hosted on this region server";
String AVG_STORE_FILE_AGE_DESC = "Average age of store files hosted on this region server";
String NUM_REFERENCE_FILES_DESC = "Number of reference file on this region server";
String STOREFILE_SIZE_DESC = "Size of storefiles being served.";
String TOTAL_REQUEST_COUNT = "totalRequestCount";
String TOTAL_REQUEST_COUNT_DESC =

20
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java
View File

@ -97,6 +97,26 @@ public interface MetricsRegionServerWrapper {
*/
long getStoreFileSize();
/**
* @return Max age of store files hosted on this region server
*/
long getMaxStoreFileAge();
/**
* @return Min age of store files hosted on this region server
*/
long getMinStoreFileAge();
/**
* @return Average age of store files hosted on this region server
*/
long getAvgStoreFileAge();
/**
* @return Number of reference files on this region server
*/
long getNumReferenceFiles();
/**
* Get the number of requests per second.
*/

20
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
View File

@ -74,6 +74,26 @@ public interface MetricsRegionWrapper {
*/
long getFilteredReadRequestCount();
/**
* @return Max age of store files under this region
*/
long getMaxStoreFileAge();
/**
* @return Min age of store files under this region
*/
long getMinStoreFileAge();
/**
* @return Average age of store files under this region
*/
long getAvgStoreFileAge();
/**
* @return Number of reference files under this region
*/
long getNumReferenceFiles();
/**
* Get the total number of mutations that have been issued against this region.
*/

8
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java
View File

@ -195,6 +195,14 @@ public class MetricsRegionServerSourceImpl
.addGauge(Interns.info(STOREFILE_COUNT, STOREFILE_COUNT_DESC), rsWrap.getNumStoreFiles())
.addGauge(Interns.info(MEMSTORE_SIZE, MEMSTORE_SIZE_DESC), rsWrap.getMemstoreSize())
.addGauge(Interns.info(STOREFILE_SIZE, STOREFILE_SIZE_DESC), rsWrap.getStoreFileSize())
.addGauge(Interns.info(MAX_STORE_FILE_AGE, MAX_STORE_FILE_AGE_DESC),
rsWrap.getMaxStoreFileAge())
.addGauge(Interns.info(MIN_STORE_FILE_AGE, MIN_STORE_FILE_AGE_DESC),
rsWrap.getMinStoreFileAge())
.addGauge(Interns.info(AVG_STORE_FILE_AGE, AVG_STORE_FILE_AGE_DESC),
rsWrap.getAvgStoreFileAge())
.addGauge(Interns.info(NUM_REFERENCE_FILES, NUM_REFERENCE_FILES_DESC),
rsWrap.getNumReferenceFiles())
.addGauge(Interns.info(RS_START_TIME_NAME, RS_START_TIME_DESC),
rsWrap.getStartCode())
.addCounter(Interns.info(TOTAL_REQUEST_COUNT, TOTAL_REQUEST_COUNT_DESC),

16
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
View File

@ -216,6 +216,22 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
MetricsRegionServerSource.MEMSTORE_SIZE_DESC),
this.regionWrapper.getMemstoreSize());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_AGE,
MetricsRegionServerSource.MAX_STORE_FILE_AGE_DESC),
this.regionWrapper.getMaxStoreFileAge());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.MIN_STORE_FILE_AGE,
MetricsRegionServerSource.MIN_STORE_FILE_AGE_DESC),
this.regionWrapper.getMinStoreFileAge());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.AVG_STORE_FILE_AGE,
MetricsRegionServerSource.AVG_STORE_FILE_AGE_DESC),
this.regionWrapper.getAvgStoreFileAge());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.NUM_REFERENCE_FILES,
MetricsRegionServerSource.NUM_REFERENCE_FILES_DESC),
this.regionWrapper.getNumReferenceFiles());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.STOREFILE_SIZE,
MetricsRegionServerSource.STOREFILE_SIZE_DESC),

20
hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
View File

@ -111,6 +111,26 @@ public class TestMetricsRegionSourceImpl {
return 0;
}
@Override
public long getMaxStoreFileAge() {
return 0;
}
@Override
public long getMinStoreFileAge() {
return 0;
}
@Override
public long getAvgStoreFileAge() {
return 0;
}
@Override
public long getNumReferenceFiles() {
return 0;
}
@Override
public long getWriteRequestCount() {
return 0;

83
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -1887,6 +1887,89 @@ public class HStore implements Store {
return this.storeEngine.getStoreFileManager().getStorefileCount();
}
@Override
public long getMaxStoreFileAge() {
long earliestTS = Long.MAX_VALUE;
for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
StoreFile.Reader r = s.getReader();
if (r == null) {
LOG.warn("StoreFile " + s + " has a null Reader");
continue;
}
if (!s.isHFile()) {
continue;
}
long createdTS = s.getFileInfo().getCreatedTimestamp();
earliestTS = (createdTS < earliestTS) ? createdTS : earliestTS;
}
long now = EnvironmentEdgeManager.currentTime();
return now - earliestTS;
}
@Override
public long getMinStoreFileAge() {
long latestTS = 0;
for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
StoreFile.Reader r = s.getReader();
if (r == null) {
LOG.warn("StoreFile " + s + " has a null Reader");
continue;
}
if (!s.isHFile()) {
continue;
}
long createdTS = s.getFileInfo().getCreatedTimestamp();
latestTS = (createdTS > latestTS) ? createdTS : latestTS;
}
long now = EnvironmentEdgeManager.currentTime();
return now - latestTS;
}
@Override
public long getAvgStoreFileAge() {
long sum = 0, count = 0;
for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
StoreFile.Reader r = s.getReader();
if (r == null) {
LOG.warn("StoreFile " + s + " has a null Reader");
continue;
}
if (!s.isHFile()) {
continue;
}
sum += s.getFileInfo().getCreatedTimestamp();
count++;
}
if (count == 0) {
return 0;
}
long avgTS = sum / count;
long now = EnvironmentEdgeManager.currentTime();
return now - avgTS;
}
@Override
public long getNumReferenceFiles() {
long numRefFiles = 0;
for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
if (s.isReference()) {
numRefFiles++;
}
}
return numRefFiles;
}
@Override
public long getNumHFiles() {
long numHFiles = 0;
for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
if (s.isHFile()) {
numHFiles++;
}
}
return numHFiles;
}
@Override
public long getStoreSizeUncompressed() {
return this.totalUncompressedBytes;

51
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java
View File

@ -66,6 +66,10 @@ class MetricsRegionServerWrapperImpl
private volatile long numStoreFiles = 0;
private volatile long memstoreSize = 0;
private volatile long storeFileSize = 0;
private volatile long maxStoreFileAge = 0;
private volatile long minStoreFileAge = 0;
private volatile long avgStoreFileAge = 0;
private volatile long numReferenceFiles = 0;
private volatile double requestsPerSecond = 0.0;
private volatile long readRequestsCount = 0;
private volatile long filteredReadRequestsCount = 0;
@ -389,6 +393,26 @@ class MetricsRegionServerWrapperImpl
return numStoreFiles;
}
@Override
public long getMaxStoreFileAge() {
return maxStoreFileAge;
}
@Override
public long getMinStoreFileAge() {
return minStoreFileAge;
}
@Override
public long getAvgStoreFileAge() {
return avgStoreFileAge;
}
@Override
public long getNumReferenceFiles() {
return numReferenceFiles;
}
@Override
public long getMemstoreSize() {
return memstoreSize;
@ -614,6 +638,9 @@ class MetricsRegionServerWrapperImpl
new HDFSBlocksDistribution();
long tempNumStores = 0, tempNumStoreFiles = 0, tempMemstoreSize = 0, tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0, tempNumReferenceFiles = 0;
long avgAgeNumerator = 0, numHFiles = 0;
long tempMinStoreFileAge = Long.MAX_VALUE;
long tempReadRequestsCount = 0, tempFilteredReadRequestsCount = 0,
tempWriteRequestsCount = 0;
long tempCheckAndMutateChecksFailed = 0;
@ -657,6 +684,20 @@ class MetricsRegionServerWrapperImpl
tempNumStoreFiles += store.getStorefilesCount();
tempMemstoreSize += store.getMemStoreSize();
tempStoreFileSize += store.getStorefilesSize();
long storeMaxStoreFileAge = store.getMaxStoreFileAge();
tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
storeMaxStoreFileAge : tempMaxStoreFileAge;
long storeMinStoreFileAge = store.getMinStoreFileAge();
tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
storeMinStoreFileAge : tempMinStoreFileAge;
long storeHFiles = store.getNumHFiles();
avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
numHFiles += storeHFiles;
tempNumReferenceFiles += store.getNumReferenceFiles();
tempStorefileIndexSize += store.getStorefilesIndexSize();
tempTotalStaticBloomSize += store.getTotalStaticBloomSize();
tempTotalStaticIndexSize += store.getTotalStaticIndexSize();
@ -723,6 +764,16 @@ class MetricsRegionServerWrapperImpl
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
if (tempMinStoreFileAge != Long.MAX_VALUE) {
minStoreFileAge = tempMinStoreFileAge;
}
if (numHFiles != 0) {
avgStoreFileAge = avgAgeNumerator / numHFiles;
}
numReferenceFiles= tempNumReferenceFiles;
readRequestsCount = tempReadRequestsCount;
filteredReadRequestsCount = tempFilteredReadRequestsCount;
writeRequestsCount = tempWriteRequestsCount;

52
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
View File

@ -43,6 +43,10 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
private long numStoreFiles;
private long memstoreSize;
private long storeFileSize;
private long maxStoreFileAge;
private long minStoreFileAge;
private long avgStoreFileAge;
private long numReferenceFiles;
private ScheduledFuture<?> regionMetricsUpdateTask;
@ -136,6 +140,26 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
return this.region.compactionsFinished.get();
}
@Override
public long getMaxStoreFileAge() {
return maxStoreFileAge;
}
@Override
public long getMinStoreFileAge() {
return minStoreFileAge;
}
@Override
public long getAvgStoreFileAge() {
return avgStoreFileAge;
}
@Override
public long getNumReferenceFiles() {
return numReferenceFiles;
}
@Override
public int getRegionHashCode() {
return this.region.hashCode();
@ -148,18 +172,46 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
long tempNumStoreFiles = 0;
long tempMemstoreSize = 0;
long tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0;
long tempMinStoreFileAge = Long.MAX_VALUE;
long tempNumReferenceFiles = 0;
long avgAgeNumerator = 0;
long numHFiles = 0;
if (region.stores != null) {
for (Store store : region.stores.values()) {
tempNumStoreFiles += store.getStorefilesCount();
tempMemstoreSize += store.getMemStoreSize();
tempStoreFileSize += store.getStorefilesSize();
long storeMaxStoreFileAge = store.getMaxStoreFileAge();
tempMaxStoreFileAge = (storeMaxStoreFileAge > tempMaxStoreFileAge) ?
storeMaxStoreFileAge : tempMaxStoreFileAge;
long storeMinStoreFileAge = store.getMinStoreFileAge();
tempMinStoreFileAge = (storeMinStoreFileAge < tempMinStoreFileAge) ?
storeMinStoreFileAge : tempMinStoreFileAge;
long storeHFiles = store.getNumHFiles();
avgAgeNumerator += store.getAvgStoreFileAge() * storeHFiles;
numHFiles += storeHFiles;
tempNumReferenceFiles += store.getNumReferenceFiles();
}
}
numStoreFiles = tempNumStoreFiles;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;
if (tempMinStoreFileAge != Long.MAX_VALUE) {
minStoreFileAge = tempMinStoreFileAge;
}
if (numHFiles != 0) {
avgStoreFileAge = avgAgeNumerator / numHFiles;
}
numReferenceFiles = tempNumReferenceFiles;
}
}

25
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
View File

@ -345,6 +345,31 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
*/
int getStorefilesCount();
/**
* @return Max age of store files in this store
*/
long getMaxStoreFileAge();
/**
* @return Min age of store files in this store
*/
long getMinStoreFileAge();
/**
* @return Average age of store files in this store, 0 if no store files
*/
long getAvgStoreFileAge();
/**
* @return Number of reference files in this store
*/
long getNumReferenceFiles();
/**
* @return Number of HFiles in this store
*/
long getNumHFiles();
/**
* @return The size of the store files, in bytes, uncompressed.
*/

7
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
View File

@ -290,6 +290,13 @@ public class StoreFile {
return this.fileInfo.isReference();
}
/**
* @return True if this is HFile.
*/
public boolean isHFile() {
return this.fileInfo.isHFile(this.fileInfo.getPath());
}
/**
* @return True if this file was made by a major compaction.
*/

12
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileInfo.java
View File

@ -97,6 +97,9 @@ public class StoreFileInfo {
private RegionCoprocessorHost coprocessorHost;
// timestamp on when the file was created, is 0 and ignored for reference or link files
private long createdTimestamp;
/**
* Create a Store File Info
* @param conf the {@link Configuration} to use
@ -132,6 +135,7 @@ public class StoreFileInfo {
" reference to " + referencePath);
} else if (isHFile(p)) {
// HFile
this.createdTimestamp = fs.getFileStatus(initialPath).getModificationTime();
this.reference = null;
this.link = null;
} else {
@ -182,6 +186,7 @@ public class StoreFileInfo {
this.fs = fs;
this.conf = conf;
this.initialPath = fileStatus.getPath();
this.createdTimestamp = fileStatus.getModificationTime();
this.reference = reference;
this.link = null;
}
@ -419,6 +424,13 @@ public class StoreFileInfo {
return m.matches() && m.groupCount() > 1;
}
/**
* @return timestamp when this file was created (as returned by filesystem)
*/
public long getCreatedTimestamp() {
return createdTimestamp;
}
/*
* Return path to the file referred to by a Reference. Presumes a directory
* hierarchy of <code>${hbase.rootdir}/data/${namespace}/tablename/regionname/familyname</code>.

20
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java
View File

@ -70,6 +70,26 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe
return 1900;
}
@Override
public long getMaxStoreFileAge() {
return 2;
}
@Override
public long getMinStoreFileAge() {
return 2;
}
@Override
public long getAvgStoreFileAge() {
return 2;
}
@Override
public long getNumReferenceFiles() {
return 2;
}
@Override
public double getRequestsPerSecond() {
return 0;

20
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
View File

@ -80,6 +80,26 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
return 107;
}
@Override
public long getMaxStoreFileAge() {
return 2;
}
@Override
public long getMinStoreFileAge() {
return 2;
}
@Override
public long getAvgStoreFileAge() {
return 2;
}
@Override
public long getNumReferenceFiles() {
return 2;
}
@Override
public long getWriteRequestCount() {
return 106;

12
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegion.java
View File

@ -42,6 +42,18 @@ public class TestMetricsRegion {
HELPER.assertGauge(
"namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_storeFileCount",
102, agg);
HELPER.assertGauge(
"namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_maxStoreFileAge",
2, agg);
HELPER.assertGauge(
"namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_minStoreFileAge",
2, agg);
HELPER.assertGauge(
"namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_avgStoreFileAge",
2, agg);
HELPER.assertGauge(
"namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_numReferenceFiles",
2, agg);
HELPER.assertGauge(
"namespace_TestNS_table_MetricsRegionWrapperStub_region_DEADBEEF001_metric_memstoreSize",
103, agg);

4
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionServer.java
View File

@ -60,6 +60,10 @@ public class TestMetricsRegionServer {
HELPER.assertGauge("regionServerStartTime", 100, serverSource);
HELPER.assertGauge("regionCount", 101, serverSource);
HELPER.assertGauge("storeCount", 2, serverSource);
HELPER.assertGauge("maxStoreFileAge", 2, serverSource);
HELPER.assertGauge("minStoreFileAge", 2, serverSource);
HELPER.assertGauge("avgStoreFileAge", 2, serverSource);
HELPER.assertGauge("numReferenceFiles", 2, serverSource);
HELPER.assertGauge("hlogFileCount", 10, serverSource);
HELPER.assertGauge("hlogFileSize", 1024000, serverSource);
HELPER.assertGauge("storeFileCount", 300, serverSource);

23
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionServerMetrics.java
View File

@ -238,6 +238,29 @@ public class TestRegionServerMetrics {
t.close();
}
@Test
public void testStoreFileAge() throws Exception {
TableName tableName = TableName.valueOf("testStoreFileAge");
byte[] cf = Bytes.toBytes("d");
byte[] row = Bytes.toBytes("rk");
byte[] qualifier = Bytes.toBytes("qual");
byte[] val = Bytes.toBytes("Value");
//Force a hfile.
Table t = TEST_UTIL.createTable(tableName, cf);
Put p = new Put(row);
p.addColumn(cf, qualifier, val);
t.put(p);
TEST_UTIL.getHBaseAdmin().flush(tableName);
metricsRegionServer.getRegionServerWrapper().forceRecompute();
assertTrue(metricsHelper.getGaugeLong("maxStoreFileAge", serverSource) > 0);
assertTrue(metricsHelper.getGaugeLong("minStoreFileAge", serverSource) > 0);
assertTrue(metricsHelper.getGaugeLong("avgStoreFileAge", serverSource) > 0);
t.close();
}
@Test
public void testCheckAndPutCount() throws Exception {
String tableNameString = "testCheckAndPutCount";