Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-25754 StripeCompactionPolicy should support compacting cold regions (#3152) 

Signed-off-by: Duo Zhang <zhangduo@apache.org>
This commit is contained in:
haxiaolin 2021-04-23 14:58:53 +08:00 committed by GitHub
parent 5f4e2e111b
commit 996862c1cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 59 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/StripeCompactionPolicy.java
View File

@ -162,7 +162,7 @@ public class StripeCompactionPolicy extends CompactionPolicy {
return filesCompacting.isEmpty()
&& (StoreUtils.hasReferences(si.getStorefiles())
|| (si.getLevel0Files().size() >= this.config.getLevel0MinFiles())
|| needsSingleStripeCompaction(si));
|| needsSingleStripeCompaction(si) || hasExpiredStripes(si));
}
@Override
@ -338,6 +338,33 @@ public class StripeCompactionPolicy extends CompactionPolicy {
return result;
}
private boolean isStripeExpired(ImmutableList<HStoreFile> storeFiles) {
long cfTtl = this.storeConfigInfo.getStoreFileTtl();
if (cfTtl == Long.MAX_VALUE) {
return false; // minversion might be set, cannot delete old files
}
long timestampCutoff = EnvironmentEdgeManager.currentTime() - cfTtl;
for (HStoreFile storeFile : storeFiles) {
// Check store file is not empty and has not expired
if (storeFile.getReader().getMaxTimestamp() >= timestampCutoff
&& storeFile.getReader().getEntries() != 0) {
return false;
}
}
return true;
}
protected boolean hasExpiredStripes(StripeInformationProvider si) {
// Find if exists a stripe where all files have expired, if any.
ArrayList<ImmutableList<HStoreFile>> stripes = si.getStripes();
for (ImmutableList<HStoreFile> stripe : stripes) {
if (isStripeExpired(stripe)) {
return true;
}
}
return false;
}
private static long getTotalKvCount(final Collection<HStoreFile> candidates) {
long totalSize = 0;
for (HStoreFile storeFile : candidates) {

31
hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/compactions/TestStripeCompactionPolicy.java
View File

@ -55,6 +55,7 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.RegionInfo;
import org.apache.hadoop.hbase.client.RegionInfoBuilder;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.io.hfile.HFile;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.hbase.regionserver.HStore;
@ -288,6 +289,35 @@ public class TestStripeCompactionPolicy {
verifyNoCompaction(policy, si);
}
@Test
public void testCheckExpiredStripeCompaction() throws Exception {
Configuration conf = HBaseConfiguration.create();
conf.setInt(StripeStoreConfig.MIN_FILES_L0_KEY, 5);
conf.setInt(StripeStoreConfig.MIN_FILES_KEY, 4);
ManualEnvironmentEdge edge = new ManualEnvironmentEdge();
long now = defaultTtl + 2;
edge.setValue(now);
EnvironmentEdgeManager.injectEdge(edge);
HStoreFile expiredFile = createFile(10), notExpiredFile = createFile(10);
when(expiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl - 1);
when(notExpiredFile.getReader().getMaxTimestamp()).thenReturn(now - defaultTtl + 1);
List<HStoreFile> expired = Lists.newArrayList(expiredFile, expiredFile);
List<HStoreFile> mixed = Lists.newArrayList(expiredFile, notExpiredFile);
StripeCompactionPolicy policy =
createPolicy(conf, defaultSplitSize, defaultSplitCount, defaultInitialCount, true);
// Merge expired if there are eligible stripes.
StripeCompactionPolicy.StripeInformationProvider si =
createStripesWithFiles(mixed, mixed, mixed);
assertFalse(policy.needsCompactions(si, al()));
si = createStripesWithFiles(mixed, mixed, mixed, expired);
assertFalse(policy.needsSingleStripeCompaction(si));
assertTrue(policy.hasExpiredStripes(si));
assertTrue(policy.needsCompactions(si, al()));
}
@Test
public void testSplitOffStripe() throws Exception {
Configuration conf = HBaseConfiguration.create();
@ -776,6 +806,7 @@ public class TestStripeCompactionPolicy {
anyBoolean())).thenReturn(mock(StoreFileScanner.class));
when(sf.getReader()).thenReturn(r);
when(sf.getBulkLoadTimestamp()).thenReturn(OptionalLong.empty());
when(r.getMaxTimestamp()).thenReturn(TimeRange.INITIAL_MAX_TIMESTAMP);
return sf;
}