From bd3b9753a9a792b402064ec4fabf2dc3c2eb41f1 Mon Sep 17 00:00:00 2001 From: zhangduo Date: Sun, 17 Apr 2016 22:18:40 +0800 Subject: [PATCH] HBASE-15665 Support using different StoreFileComparators for different CompactionPolicies --- .../regionserver/DateTieredStoreEngine.java | 5 ++-- .../regionserver/DefaultStoreEngine.java | 5 ++-- .../regionserver/DefaultStoreFileManager.java | 24 ++++++++++++------- .../hadoop/hbase/regionserver/HStore.java | 5 ++-- .../hbase/regionserver/StoreFileManager.java | 13 +++++++--- .../regionserver/StripeStoreFileManager.java | 6 +++++ .../DateTieredCompactionPolicy.java | 2 -- 7 files changed, 41 insertions(+), 19 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java index 773baabbcdb..2d86e396562 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DateTieredStoreEngine.java @@ -57,8 +57,9 @@ public class DateTieredStoreEngine extends StoreEngine storeFileComparator; /** * List of store files inside this store. This is an immutable list that * is atomically replaced when its contents change. @@ -62,9 +63,11 @@ class DefaultStoreFileManager implements StoreFileManager { */ private volatile List compactedfiles = null; - public DefaultStoreFileManager(CellComparator kvComparator, Configuration conf, + public DefaultStoreFileManager(CellComparator kvComparator, + Comparator storeFileComparator, Configuration conf, CompactionConfiguration comConf) { this.kvComparator = kvComparator; + this.storeFileComparator = storeFileComparator; this.comConf = comConf; this.blockingFileCount = conf.getInt(HStore.BLOCKING_STOREFILES_KEY, HStore.DEFAULT_BLOCKING_STOREFILE_COUNT); @@ -210,13 +213,13 @@ class DefaultStoreFileManager implements StoreFileManager { } private void sortAndSetStoreFiles(List storeFiles) { - Collections.sort(storeFiles, StoreFile.Comparators.SEQ_ID); + Collections.sort(storeFiles, storeFileComparator); storefiles = ImmutableList.copyOf(storeFiles); } private List sortCompactedfiles(List storefiles) { // Sorting may not be really needed here for the compacted files? - Collections.sort(storefiles, StoreFile.Comparators.SEQ_ID); + Collections.sort(storefiles, storeFileComparator); return new ArrayList(storefiles); } @@ -229,5 +232,10 @@ class DefaultStoreFileManager implements StoreFileManager { } return (double) (storefileCount - minFilesToCompact) / (blockingFileCount - minFilesToCompact); } + + @Override + public Comparator getStoreFileComparator() { + return storeFileComparator; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 7468be0c722..c02627f0bc7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -1466,7 +1466,8 @@ public class HStore implements Store { filesToCompact = filesToCompact.subList(count - N, count); isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount()); filesCompacting.addAll(filesToCompact); - Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID); + Collections.sort(filesCompacting, storeEngine.getStoreFileManager() + .getStoreFileComparator()); } } finally { this.lock.readLock().unlock(); @@ -1655,7 +1656,7 @@ public class HStore implements Store { Preconditions.checkArgument(false, "%s overlaps with %s", filesToAdd, filesCompacting); } filesCompacting.addAll(filesToAdd); - Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID); + Collections.sort(filesCompacting, storeEngine.getStoreFileManager().getStoreFileComparator()); } private void removeUnneededFiles() throws IOException { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java index 7e7054766a4..cb192678c6f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileManager.java @@ -20,14 +20,15 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; import java.util.Collection; +import java.util.Comparator; import java.util.Iterator; import java.util.List; -import org.apache.hadoop.hbase.classification.InterfaceAudience; +import com.google.common.collect.ImmutableCollection; + import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; - -import com.google.common.collect.ImmutableCollection; +import org.apache.hadoop.hbase.classification.InterfaceAudience; /** * Manages the store files and basic metadata about that that determines the logical structure @@ -163,4 +164,10 @@ public interface StoreFileManager { * @see Store#getCompactionPressure() */ double getCompactionPressure(); + + /** + * @return the comparator used to sort storefiles. Usually, the + * {@link StoreFile#getMaxSequenceId()} is the first priority. + */ + Comparator getStoreFileComparator(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java index ef2c282424c..df1ddf2d839 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFileManager.java @@ -23,6 +23,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; import java.util.List; @@ -1072,4 +1073,9 @@ public class StripeStoreFileManager } return max; } + + @Override + public Comparator getStoreFileComparator() { + return StoreFile.Comparators.SEQ_ID; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java index 6527c9b214f..669820c44b1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/compactions/DateTieredCompactionPolicy.java @@ -191,10 +191,8 @@ public class DateTieredCompactionPolicy extends SortedCompactionPolicy { long now = EnvironmentEdgeManager.currentTime(); long oldestToCompact = getOldestToCompact(comConf.getMaxStoreFileAgeMillis(), now); - // Make sure the store files is sorted by SeqId then maxTimestamp List storeFileList = Lists.newArrayList(filterOldStoreFiles(candidateSelection, oldestToCompact)); - Collections.sort(storeFileList, StoreFile.Comparators.SEQ_ID_MAX_TIMESTAMP); List> storefileMaxTimestampPairs = Lists.newArrayListWithCapacity(Iterables.size(storeFileList));