Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-23590 : Update maxStoreFileRefCount to maxCompactedStoreFileRefCount for auto region recovery based on old reader references 

Signed-off-by: Anoop Sam John <anoopsamjohn@apache.org>
This commit is contained in:
Viraj Jasani 2020-01-01 21:57:03 +05:30
parent 5a0dd574a6
commit 17652a7b32
No known key found for this signature in database
GPG Key ID: E906DFF511D3E5DB
18 changed files with 108 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionLoad.java
View File

@ -366,8 +366,8 @@ public class RegionLoad implements RegionMetrics {
}
@Override
public int getMaxStoreFileRefCount() {
return metrics.getMaxStoreFileRefCount();
public int getMaxCompactedStoreFileRefCount() {
return metrics.getMaxCompactedStoreFileRefCount();
}
/**

4
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetrics.java
View File

@ -150,8 +150,8 @@ public interface RegionMetrics {
int getStoreRefCount();
/**
* @return the max reference count for any store file among all stores files
* @return the max reference count for any store file among all compacted stores files
* of this region
*/
int getMaxStoreFileRefCount();
int getMaxCompactedStoreFileRefCount();
}

26
hbase-client/src/main/java/org/apache/hadoop/hbase/RegionMetricsBuilder.java
View File

@ -65,7 +65,7 @@ public final class RegionMetricsBuilder {
.setStoreCount(regionLoadPB.getStores())
.setStoreFileCount(regionLoadPB.getStorefiles())
.setStoreRefCount(regionLoadPB.getStoreRefCount())
.setMaxStoreFileRefCount(regionLoadPB.getMaxStoreFileRefCount())
.setMaxCompactedStoreFileRefCount(regionLoadPB.getMaxCompactedStoreFileRefCount())
.setStoreFileSize(new Size(regionLoadPB.getStorefileSizeMB(), Size.Unit.MEGABYTE))
.setStoreSequenceIds(regionLoadPB.getStoreCompleteSequenceIdList().stream()
.collect(Collectors.toMap(
@ -112,7 +112,7 @@ public final class RegionMetricsBuilder {
.setStores(regionMetrics.getStoreCount())
.setStorefiles(regionMetrics.getStoreFileCount())
.setStoreRefCount(regionMetrics.getStoreRefCount())
.setMaxStoreFileRefCount(regionMetrics.getMaxStoreFileRefCount())
.setMaxCompactedStoreFileRefCount(regionMetrics.getMaxCompactedStoreFileRefCount())
.setStorefileSizeMB((int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE))
.addAllStoreCompleteSequenceId(toStoreSequenceId(regionMetrics.getStoreSequenceId()))
.setStoreUncompressedSizeMB(
@ -128,7 +128,7 @@ public final class RegionMetricsBuilder {
private int storeCount;
private int storeFileCount;
private int storeRefCount;
private int maxStoreFileRefCount;
private int maxCompactedStoreFileRefCount;
private long compactingCellCount;
private long compactedCellCount;
private Size storeFileSize = Size.ZERO;
@ -161,8 +161,8 @@ public final class RegionMetricsBuilder {
this.storeRefCount = value;
return this;
}
public RegionMetricsBuilder setMaxStoreFileRefCount(int value) {
this.maxStoreFileRefCount = value;
public RegionMetricsBuilder setMaxCompactedStoreFileRefCount(int value) {
this.maxCompactedStoreFileRefCount = value;
return this;
}
public RegionMetricsBuilder setCompactingCellCount(long value) {
@ -235,7 +235,7 @@ public final class RegionMetricsBuilder {
storeCount,
storeFileCount,
storeRefCount,
maxStoreFileRefCount,
maxCompactedStoreFileRefCount,
compactingCellCount,
compactedCellCount,
storeFileSize,
@ -259,7 +259,7 @@ public final class RegionMetricsBuilder {
private final int storeCount;
private final int storeFileCount;
private final int storeRefCount;
private final int maxStoreFileRefCount;
private final int maxCompactedStoreFileRefCount;
private final long compactingCellCount;
private final long compactedCellCount;
private final Size storeFileSize;
@ -280,7 +280,7 @@ public final class RegionMetricsBuilder {
int storeCount,
int storeFileCount,
int storeRefCount,
int maxStoreFileRefCount,
int maxCompactedStoreFileRefCount,
final long compactingCellCount,
long compactedCellCount,
Size storeFileSize,
@ -301,7 +301,7 @@ public final class RegionMetricsBuilder {
this.storeCount = storeCount;
this.storeFileCount = storeFileCount;
this.storeRefCount = storeRefCount;
this.maxStoreFileRefCount = maxStoreFileRefCount;
this.maxCompactedStoreFileRefCount = maxCompactedStoreFileRefCount;
this.compactingCellCount = compactingCellCount;
this.compactedCellCount = compactedCellCount;
this.storeFileSize = Preconditions.checkNotNull(storeFileSize);
@ -341,8 +341,8 @@ public final class RegionMetricsBuilder {
}
@Override
public int getMaxStoreFileRefCount() {
return maxStoreFileRefCount;
public int getMaxCompactedStoreFileRefCount() {
return maxCompactedStoreFileRefCount;
}
@Override
@ -433,8 +433,8 @@ public final class RegionMetricsBuilder {
this.getStoreFileCount());
Strings.appendKeyValue(sb, "storeRefCount",
this.getStoreRefCount());
Strings.appendKeyValue(sb, "maxStoreFileRefCount",
this.getMaxStoreFileRefCount());
Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
this.getMaxCompactedStoreFileRefCount());
Strings.appendKeyValue(sb, "uncompressedStoreFileSize",
this.getUncompressedStoreFileSize());
Strings.appendKeyValue(sb, "lastMajorCompactionTimestamp",

10
hbase-client/src/main/java/org/apache/hadoop/hbase/ServerMetricsBuilder.java
View File

@ -360,7 +360,7 @@ public final class ServerMetricsBuilder {
int storeCount = 0;
int storeFileCount = 0;
int storeRefCount = 0;
int maxStoreFileRefCount = 0;
int maxCompactedStoreFileRefCount = 0;
long uncompressedStoreFileSizeMB = 0;
long storeFileSizeMB = 0;
long memStoreSizeMB = 0;
@ -376,8 +376,9 @@ public final class ServerMetricsBuilder {
storeCount += r.getStoreCount();
storeFileCount += r.getStoreFileCount();
storeRefCount += r.getStoreRefCount();
int currentMaxStoreFileRefCount = r.getMaxStoreFileRefCount();
maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount);
int currentMaxCompactedStoreFileRefCount = r.getMaxCompactedStoreFileRefCount();
maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
currentMaxCompactedStoreFileRefCount);
uncompressedStoreFileSizeMB += r.getUncompressedStoreFileSize().get(Size.Unit.MEGABYTE);
storeFileSizeMB += r.getStoreFileSize().get(Size.Unit.MEGABYTE);
memStoreSizeMB += r.getMemStoreSize().get(Size.Unit.MEGABYTE);
@ -399,7 +400,8 @@ public final class ServerMetricsBuilder {
Strings.appendKeyValue(sb, "numberOfStores", storeCount);
Strings.appendKeyValue(sb, "numberOfStorefiles", storeFileCount);
Strings.appendKeyValue(sb, "storeRefCount", storeRefCount);
Strings.appendKeyValue(sb, "maxStoreFileRefCount", maxStoreFileRefCount);
Strings.appendKeyValue(sb, "maxCompactedStoreFileRefCount",
maxCompactedStoreFileRefCount);
Strings.appendKeyValue(sb, "storefileUncompressedSizeMB", uncompressedStoreFileSizeMB);
Strings.appendKeyValue(sb, "storefileSizeMB", storeFileSizeMB);
if (uncompressedStoreFileSizeMB != 0) {

32
hbase-common/src/main/resources/hbase-default.xml
View File

@ -1891,19 +1891,25 @@ possible configurations would overwhelm and obscure the important.
<name>hbase.regions.recovery.store.file.ref.count</name>
<value>-1</value>
<description>
Very large ref count on a file indicates
that it is a ref leak on that object. Such files
can not be removed even after it is invalidated
via compaction. Only way to recover in such
scenario is to reopen the region which can
release all resources, like the refcount, leases, etc.
This config represents Store files Ref Count threshold
value considered for reopening regions.
Any region with store files ref count > this value
would be eligible for reopening by master.
Default value -1 indicates this feature is turned off.
Only positive integer value should be provided to enable
this feature.
Very large number of ref count on a compacted
store file indicates that it is a ref leak
on that object(compacted store file).
Such files can not be removed after
it is invalidated via compaction.
Only way to recover in such scenario is to
reopen the region which can release
all resources, like the refcount,
leases, etc. This config represents Store files Ref
Count threshold value considered for reopening
regions. Any region with compacted store files
ref count > this value would be eligible for
reopening by master. Here, we get the max
refCount among all refCounts on all
compacted away store files that belong to a
particular region. Default value -1 indicates
this feature is turned off. Only positive
integer value should be provided to
enable this feature.
</description>
</property>
</configuration>

2
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java
View File

@ -233,7 +233,7 @@ public interface MetricsRegionServerSource extends BaseSource, JvmPauseMonitorSo
String STOREFILE_COUNT_DESC = "Number of Store Files";
String STORE_REF_COUNT = "storeRefCount";
String STORE_REF_COUNT_DESC = "Store reference count";
String MAX_STORE_FILE_REF_COUNT = "maxStoreFileRefCount";
String MAX_COMPACTED_STORE_FILE_REF_COUNT = "maxCompactedStoreFileRefCount";
String MEMSTORE_SIZE = "memStoreSize";
String MEMSTORE_SIZE_DESC = "Size of the memstore";
String STOREFILE_SIZE = "storeFileSize";

4
hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapper.java
View File

@ -160,7 +160,7 @@ public interface MetricsRegionWrapper {
/**
* @return the max number of references active on any store file among
* all store files that belong to this region
* all compacted store files that belong to this region
*/
long getMaxStoreFileRefCount();
long getMaxCompactedStoreFileRefCount();
}

7
hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionSourceImpl.java
View File

@ -218,9 +218,10 @@ public class MetricsRegionSourceImpl implements MetricsRegionSource {
MetricsRegionServerSource.STORE_REF_COUNT),
this.regionWrapper.getStoreRefCount());
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT,
MetricsRegionServerSource.MAX_STORE_FILE_REF_COUNT),
this.regionWrapper.getMaxStoreFileRefCount());
regionNamePrefix + MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT,
MetricsRegionServerSource.MAX_COMPACTED_STORE_FILE_REF_COUNT),
this.regionWrapper.getMaxCompactedStoreFileRefCount()
);
mrb.addGauge(Interns.info(
regionNamePrefix + MetricsRegionServerSource.MEMSTORE_SIZE,
MetricsRegionServerSource.MEMSTORE_SIZE_DESC),

2
hbase-hadoop2-compat/src/test/java/org/apache/hadoop/hbase/regionserver/TestMetricsRegionSourceImpl.java
View File

@ -100,7 +100,7 @@ public class TestMetricsRegionSourceImpl {
}
@Override
public long getMaxStoreFileRefCount() {
public long getMaxCompactedStoreFileRefCount() {
return 0;
}

4
hbase-protocol-shaded/src/main/protobuf/ClusterStatus.proto
View File

@ -151,10 +151,10 @@ message RegionLoad {
optional int32 store_ref_count = 21 [default = 0];
/**
* The max number of references active on single store file among all store files
* The max number of references active on single store file among all compacted store files
* that belong to given region
*/
optional int32 max_store_file_ref_count = 22 [default = 0];
optional int32 max_compacted_store_file_ref_count = 22 [default = 0];
}
message UserLoad {

4
hbase-protocol/src/main/protobuf/ClusterStatus.proto
View File

@ -147,10 +147,10 @@ message RegionLoad {
optional int32 store_ref_count = 21 [default = 0];
/**
* The max number of references active on single store file among all store files
* The max number of references active on single store file among all compacted store files
* that belong to given region
*/
optional int32 max_store_file_ref_count = 22 [default = 0];
optional int32 max_compacted_store_file_ref_count = 22 [default = 0];
}
message UserLoad {

17
hbase-server/src/main/java/org/apache/hadoop/hbase/master/RegionsRecoveryChore.java
View File

@ -130,17 +130,18 @@ public class RegionsRecoveryChore extends ScheduledChore {
for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
// For each region, each store file can have different ref counts
// We need to find maximum of all such ref counts and if that max count
// is beyond a threshold value, we should reopen the region.
// Here, we take max ref count of all store files and not the cumulative
// count of all store files
final int maxStoreFileRefCount = regionMetrics.getMaxStoreFileRefCount();
// For each region, each compacted store file can have different ref counts
// We need to find maximum of all such ref counts and if that max count of compacted
// store files is beyond a threshold value, we should reopen the region.
// Here, we take max ref count of all compacted store files and not the cumulative
// count of all compacted store files
final int maxCompactedStoreFileRefCount = regionMetrics
.getMaxCompactedStoreFileRefCount();
if (maxStoreFileRefCount > storeFileRefCountThreshold) {
if (maxCompactedStoreFileRefCount > storeFileRefCountThreshold) {
final byte[] regionName = regionMetrics.getRegionName();
prepareTableToReopenRegionsMap(tableToReopenRegionsMap, regionName,
maxStoreFileRefCount);
maxCompactedStoreFileRefCount);
}
}
}

9
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
View File

@ -1660,7 +1660,7 @@ public class HRegionServer extends HasThread implements
int stores = 0;
int storefiles = 0;
int storeRefCount = 0;
int maxStoreFileRefCount = 0;
int maxCompactedStoreFileRefCount = 0;
int storeUncompressedSizeMB = 0;
int storefileSizeMB = 0;
int memstoreSizeMB = (int) (r.getMemStoreDataSize() / 1024 / 1024);
@ -1676,8 +1676,9 @@ public class HRegionServer extends HasThread implements
storefiles += store.getStorefilesCount();
int currentStoreRefCount = store.getStoreRefCount();
storeRefCount += currentStoreRefCount;
int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
maxStoreFileRefCount = Math.max(maxStoreFileRefCount, currentMaxStoreFileRefCount);
int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
maxCompactedStoreFileRefCount = Math.max(maxCompactedStoreFileRefCount,
currentMaxCompactedStoreFileRefCount);
storeUncompressedSizeMB += (int) (store.getStoreSizeUncompressed() / 1024 / 1024);
storefileSizeMB += (int) (store.getStorefilesSize() / 1024 / 1024);
//TODO: storefileIndexSizeKB is same with rootLevelIndexSizeKB?
@ -1706,7 +1707,7 @@ public class HRegionServer extends HasThread implements
.setStores(stores)
.setStorefiles(storefiles)
.setStoreRefCount(storeRefCount)
.setMaxStoreFileRefCount(maxStoreFileRefCount)
.setMaxCompactedStoreFileRefCount(maxCompactedStoreFileRefCount)
.setStoreUncompressedSizeMB(storeUncompressedSizeMB)
.setStorefileSizeMB(storefileSizeMB)
.setMemStoreSizeMB(memstoreSizeMB)

11
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
View File

@ -2779,18 +2779,19 @@ public class HStore implements Store, HeapSize, StoreConfigInformation, Propagat
}
/**
* @return get maximum ref count of storeFile among all HStore Files
* @return get maximum ref count of storeFile among all compacted HStore Files
* for the HStore
*/
public int getMaxStoreFileRefCount() {
OptionalInt maxStoreFileRefCount = this.storeEngine.getStoreFileManager()
.getStorefiles()
public int getMaxCompactedStoreFileRefCount() {
OptionalInt maxCompactedStoreFileRefCount = this.storeEngine.getStoreFileManager()
.getCompactedfiles()
.stream()
.filter(sf -> sf.getReader() != null)
.filter(HStoreFile::isHFile)
.mapToInt(HStoreFile::getRefCount)
.max();
return maxStoreFileRefCount.isPresent() ? maxStoreFileRefCount.getAsInt() : 0;
return maxCompactedStoreFileRefCount.isPresent()
? maxCompactedStoreFileRefCount.getAsInt() : 0;
}
void reportArchivedFilesForQuota(List<? extends StoreFile> archivedFiles, List<Long> fileSizes) {

16
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperImpl.java
View File

@ -49,7 +49,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
private Runnable runnable;
private long numStoreFiles;
private long storeRefCount;
private long maxStoreFileRefCount;
private long maxCompactedStoreFileRefCount;
private long memstoreSize;
private long storeFileSize;
private long maxStoreFileAge;
@ -127,8 +127,8 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
}
@Override
public long getMaxStoreFileRefCount() {
return maxStoreFileRefCount;
public long getMaxCompactedStoreFileRefCount() {
return maxCompactedStoreFileRefCount;
}
@Override
@ -234,7 +234,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
public void run() {
long tempNumStoreFiles = 0;
int tempStoreRefCount = 0;
int tempMaxStoreFileRefCount = 0;
int tempMaxCompactedStoreFileRefCount = 0;
long tempMemstoreSize = 0;
long tempStoreFileSize = 0;
long tempMaxStoreFileAge = 0;
@ -249,9 +249,9 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
tempNumStoreFiles += store.getStorefilesCount();
int currentStoreRefCount = store.getStoreRefCount();
tempStoreRefCount += currentStoreRefCount;
int currentMaxStoreFileRefCount = store.getMaxStoreFileRefCount();
tempMaxStoreFileRefCount = Math.max(tempMaxStoreFileRefCount,
currentMaxStoreFileRefCount);
int currentMaxCompactedStoreFileRefCount = store.getMaxCompactedStoreFileRefCount();
tempMaxCompactedStoreFileRefCount = Math.max(tempMaxCompactedStoreFileRefCount,
currentMaxCompactedStoreFileRefCount);
tempMemstoreSize += store.getMemStoreSize().getDataSize();
tempStoreFileSize += store.getStorefilesSize();
OptionalLong storeMaxStoreFileAge = store.getMaxStoreFileAge();
@ -279,7 +279,7 @@ public class MetricsRegionWrapperImpl implements MetricsRegionWrapper, Closeable
numStoreFiles = tempNumStoreFiles;
storeRefCount = tempStoreRefCount;
maxStoreFileRefCount = tempMaxStoreFileRefCount;
maxCompactedStoreFileRefCount = tempMaxCompactedStoreFileRefCount;
memstoreSize = tempMemstoreSize;
storeFileSize = tempStoreFileSize;
maxStoreFileAge = tempMaxStoreFileAge;

12
hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRegionsRecoveryChore.java
View File

@ -214,7 +214,7 @@ public class TestRegionsRecoveryChore {
Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(),
Mockito.anyLong(), Mockito.anyLong());
// default maxStoreFileRefCount is -1 (no regions to be reopened using AM)
// default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM)
Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
Mockito.verify(assignmentManager, Mockito.times(0))
.getRegionInfo(Mockito.any());
@ -380,7 +380,7 @@ public class TestRegionsRecoveryChore {
return serverMetrics;
}
private static RegionMetrics getRegionMetrics(byte[] regionName, int storeRefCount) {
private static RegionMetrics getRegionMetrics(byte[] regionName, int compactedStoreRefCount) {
RegionMetrics regionMetrics = new RegionMetrics() {
@Override
@ -480,12 +480,12 @@ public class TestRegionsRecoveryChore {
@Override
public int getStoreRefCount() {
return storeRefCount;
return compactedStoreRefCount;
}
@Override
public int getMaxStoreFileRefCount() {
return storeRefCount;
public int getMaxCompactedStoreFileRefCount() {
return compactedStoreRefCount;
}
};
@ -610,4 +610,4 @@ public class TestRegionsRecoveryChore {
}
}
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionWrapperStub.java
View File

@ -66,7 +66,7 @@ public class MetricsRegionWrapperStub implements MetricsRegionWrapper {
}
@Override
public long getMaxStoreFileRefCount() {
public long getMaxCompactedStoreFileRefCount() {
return 0;
}

32
src/main/asciidoc/_chapters/hbase-default.adoc
View File

@ -2047,19 +2047,25 @@ A comma-separated list of
+
.Description
Very large ref count on a file indicates
that it is a ref leak on that object. Such files
can not be removed even after it is invalidated
via compaction. Only way to recover in such
scenario is to reopen the region which can
release all resources, like the refcount, leases, etc.
This config represents Store files Ref Count threshold
value considered for reopening regions.
Any region with store files ref count > this value
would be eligible for reopening by master.
Default value -1 indicates this feature is turned off.
Only positive integer value should be provided to enable
this feature.
Very large number of ref count on a compacted
store file indicates that it is a ref leak
on that object(compacted store file).
Such files can not be removed after
it is invalidated via compaction.
Only way to recover in such scenario is to
reopen the region which can release
all resources, like the refcount,
leases, etc. This config represents Store files Ref
Count threshold value considered for reopening
regions. Any region with compacted store files
ref count > this value would be eligible for
reopening by master. Here, we get the max
refCount among all refCounts on all
compacted away store files that belong to a
particular region. Default value -1 indicates
this feature is turned off. Only positive
integer value should be provided to
enable this feature.
+
.Default