Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-22459 Expose store reader reference count (#248)

This commit is contained in:
Andrew Purtell 2019-05-30 15:04:16 -07:00 committed by GitHub
parent 4657c90991
commit 98a1552687
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 73 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
View File

@ -363,6 +363,13 @@ public class RegionLoad implements RegionMetrics {
return metrics.getLastMajorCompactionTimestamp();
}
/**
* @return the reference count for the stores of this region
*/
public int getStoreRefCount() {
return metrics.getStoreRefCount();
}
/**
* @see java.lang.Object#toString()
*/
@ -371,6 +378,7 @@ public class RegionLoad implements RegionMetrics {
StringBuilder sb = Strings.appendKeyValue(new StringBuilder(), "numberOfStores",
this.getStores());
Strings.appendKeyValue(sb, "numberOfStorefiles", this.getStorefiles());
Strings.appendKeyValue(sb, "storeRefCount", this.getStoreRefCount());
Strings.appendKeyValue(sb, "storefileUncompressedSizeMB",
this.getStoreUncompressedSizeMB());
Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",

4
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
View File

@ -150,4 +150,8 @@ public interface RegionMetrics {
*/
long getLastMajorCompactionTimestamp();
/**
* @return the reference count for the stores of this region
*/
int getStoreRefCount();
}

18
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
View File

@ -65,6 +65,7 @@ public final class RegionMetricsBuilder {
Size.Unit.KILOBYTE))
.setStoreCount(regionLoadPB.getStores())
.setStoreFileCount(regionLoadPB.getStorefiles())
.setStoreRefCount(regionLoadPB.getStoreRefCount())
.setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(), Size.Unit.MEGABYTE))
.setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream()
.collect(Collectors.toMap(
@ -111,6 +112,7 @@ public final class RegionMetricsBuilder {
.get(Size.Unit.KILOBYTE))
.setStores(regionMetrics.getStoreCount())
.setStorefiles(regionMetrics.getStoreCount())
.setStoreRefCount(regionMetrics.getStoreRefCount())
.setStorefileSizeMB((int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE))
.addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId()))
.setStoreUncompressedSizeMB(
@ -125,6 +127,7 @@ public final class RegionMetricsBuilder {
private final byte[] name;
private int storeCount;
private int storeFileCount;
private int storeRefCount;
private long compactingCellCount;
private long compactedCellCount;
private Size storeFileSize = Size.ZERO;
@ -154,6 +157,10 @@ public final class RegionMetricsBuilder {
this.storeFileCount = value;
return this;
}
public RegionMetricsBuilder setStoreRefCount(int value) {
this.storeRefCount = value;
return this;
}
public RegionMetricsBuilder setCompactingCellCount(long value) {
this.compactingCellCount = value;
return this;
@ -227,6 +234,7 @@ public final class RegionMetricsBuilder {
return new RegionMetricsImpl(name,
storeCount,
storeFileCount,
storeRefCount,
compactingCellCount,
compactedCellCount,
storeFileSize,
@ -250,6 +258,7 @@ public final class RegionMetricsBuilder {
private final byte[] name;
private final int storeCount;
private final int storeFileCount;
private final int storeRefCount;
private final long compactingCellCount;
private final long compactedCellCount;
private final Size storeFileSize;
@ -270,6 +279,7 @@ public final class RegionMetricsBuilder {
RegionMetricsImpl(byte[] name,
int storeCount,
int storeFileCount,
int storeRefCount,
final long compactingCellCount,
long compactedCellCount,
Size storeFileSize,
@ -290,6 +300,7 @@ public final class RegionMetricsBuilder {
this.name = Preconditions.checkNotNull(name);
this.storeCount = storeCount;
this.storeFileCount = storeFileCount;
this.storeRefCount = storeRefCount;
this.compactingCellCount = compactingCellCount;
this.compactedCellCount = compactedCellCount;
this.storeFileSize = Preconditions.checkNotNull(storeFileSize);
@ -324,6 +335,11 @@ public final class RegionMetricsBuilder {
return storeFileCount;
}
@Override
public int getStoreRefCount() {
return storeRefCount;
}
@Override
public Size getStoreFileSize() {
return storeFileSize;
@ -415,6 +431,8 @@ public final class RegionMetricsBuilder {
this.getStoreCount());
Strings.appendKeyValue(sb, "storeFileCount",
this.getStoreFileCount());
Strings.appendKeyValue(sb, "storeRefCount",
this.getStoreRefCount());
Strings.appendKeyValue(sb, "uncompressedStoreFileSize",
this.getUncompressedStoreFileSize());
Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",

2
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
View File

@ -231,6 +231,8 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String WALFILE_SIZE_DESC = "Size of all WAL Files";
String STOREFILE_COUNT = "storeFileCount";
String STOREFILE_COUNT_DESC = "Number of Store Files";
String STORE_REF_COUNT = "storeRefCount";
String STORE_REF_COUNT_DESC = "Store reference count";
String MEMSTORE_SIZE = "memStoreSize";
String MEMSTORE_SIZE_DESC = "Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";

5
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
View File

@ -159,4 +159,9 @@ public interface MetricsRegionWrapper {
* Get the replica id of this region.
*/
int getReplicaId();
/**
* @return the number of references active on the store
*/
long getStoreRefCount();
}

4
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
View File

@ -213,6 +213,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
regionNamePrefix + MetricsRegionServerSource.STOREFILE_COUNT,
MetricsRegionServerSource.STOREFILE_COUNT_DESC),
this.regionWrapper.getNumStoreFiles());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.STORE_REF_COUNT,
MetricsRegionServerSource.STORE_REF_COUNT),
this.regionWrapper.getStoreRefCount());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
MetricsRegionServerSource.MEMSTORE_SIZE_DESC),

5
hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
View File

@ -94,6 +94,11 @@ public class TestMetricsRegionSourceImpl {
return 0;
}
@Override
public long getStoreRefCount() {
return 0;
}
@Override
public long getMemStoreSize() {
return 0;

3
hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
View File

@ -146,6 +146,9 @@ message RegionLoad {
/** the current total coprocessor requests made to region */
optional uint64 cp_requests_count = 20;
/** the number of references active on the store */
optional int32 store_ref_count = 21 [default = 0];
}
/* Server-level protobufs */

3
hbase-protocol/src/main/protobuf/ClusterStatus.proto
View File

@ -142,6 +142,9 @@ message RegionLoad {
/** the current total coprocessor requests made to region */
optional uint64 cp_requests_count = 20;
/** the number of references active on the store */
optional int32 store_ref_count = 21 [default = 0];
}
/* Server-level protobufs */

6
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -2776,4 +2776,10 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
public int getCurrentParallelPutCount() {
return currentParallelPutCount.get();
}
public int getStoreRefCount() {
return this.storeEngine.getStoreFileManager().getStorefiles().stream()
.filter(sf -> sf.getReader() != null).filter(HStoreFile::isHFile)
.mapToInt(HStoreFile::getRefCount).sum();
}
}

11
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
View File

@ -48,6 +48,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
private ScheduledExecutorService executor;
private Runnable runnable;
private long numStoreFiles;
private long storeRefCount;
private long memstoreSize;
private long storeFileSize;
private long maxStoreFileAge;
@ -119,6 +120,11 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
return storeFileSize;
}
@Override
public long getStoreRefCount() {
return storeRefCount;
}
@Override
public long getReadRequestCount() {
return this.region.getReadRequestsCount();
@ -226,6 +232,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
@Override
public void run() {
long tempNumStoreFiles = 0;
int tempStoreRefCount = 0;
long tempMemstoreSize = 0;
long tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0;
@ -237,8 +244,9 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
long avgAgeNumerator = 0;
long numHFiles = 0;
if (region.stores != null) {
for (Store store : region.stores.values()) {
for (HStore store : region.stores.values()) {
tempNumStoreFiles += store.getStorefilesCount();
tempStoreRefCount += store.getStoreRefCount();
tempMemstoreSize += store.getMemStoreSize().getDataSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@ -265,6 +273,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
}
numStoreFiles = tempNumStoreFiles;
storeRefCount = tempStoreRefCount;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;

5
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
View File

@ -60,6 +60,11 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
return 102;
}
@Override
public long getStoreRefCount() {
return 0;
}
@Override
public long getMemStoreSize() {
return 103;