HBASE-7935 make policy and compactor in default store engine separately pluggable (for things like tier-based, and default policy experiments with permutations)
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1453586 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
sershe
parent
6ae6e5559b
commit
e67d82a2a0
|
|
@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.regionserver.compactions.CompactionPolicy;
|
|||
import org.apache.hadoop.hbase.regionserver.compactions.Compactor;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
|
||||
import org.apache.hadoop.hbase.util.ReflectionUtils;
|
||||
|
||||
/**
|
||||
* Default StoreEngine creates the default compactor, policy, and store file manager, or
|
||||
|
|
@ -40,21 +41,40 @@ import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
|
|||
public class DefaultStoreEngine extends StoreEngine<
|
||||
DefaultCompactionPolicy, DefaultCompactor, DefaultStoreFileManager> {
|
||||
|
||||
public DefaultStoreEngine(Configuration conf, Store store, KVComparator comparator) {
|
||||
super(conf, store, comparator);
|
||||
public static final String DEFAULT_COMPACTOR_CLASS_KEY =
|
||||
"hbase.hstore.defaultengine.compactor.class";
|
||||
public static final String DEFAULT_COMPACTION_POLICY_CLASS_KEY =
|
||||
"hbase.hstore.defaultengine.compactionpolicy.class";
|
||||
|
||||
private static final Class<? extends DefaultCompactor>
|
||||
DEFAULT_COMPACTOR_CLASS = DefaultCompactor.class;
|
||||
private static final Class<? extends DefaultCompactionPolicy>
|
||||
DEFAULT_COMPACTION_POLICY_CLASS = DefaultCompactionPolicy.class;
|
||||
|
||||
@Override
|
||||
protected void createComponents(
|
||||
Configuration conf, Store store, KVComparator kvComparator) throws IOException {
|
||||
storeFileManager = new DefaultStoreFileManager(kvComparator, conf);
|
||||
String className = conf.get(DEFAULT_COMPACTOR_CLASS_KEY, DEFAULT_COMPACTOR_CLASS.getName());
|
||||
try {
|
||||
compactor = ReflectionUtils.instantiateWithCustomCtor(className,
|
||||
new Class[] { Configuration.class, Store.class }, new Object[] { conf, store });
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Unable to load configured compactor '" + className + "'", e);
|
||||
}
|
||||
className = conf.get(
|
||||
DEFAULT_COMPACTION_POLICY_CLASS_KEY, DEFAULT_COMPACTION_POLICY_CLASS.getName());
|
||||
try {
|
||||
compactionPolicy = ReflectionUtils.instantiateWithCustomCtor(className,
|
||||
new Class[] { Configuration.class, StoreConfigInformation.class },
|
||||
new Object[] { conf, store });
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Unable to load configured compaction policy '" + className + "'", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected void createComponents() {
|
||||
storeFileManager = new DefaultStoreFileManager(this.comparator, this.conf);
|
||||
|
||||
// TODO: compactor and policy may be separately pluggable, but must derive from default ones.
|
||||
compactor = new DefaultCompactor(this.conf, this.store);
|
||||
compactionPolicy = new DefaultCompactionPolicy(this.conf, this.store/*as StoreConfigInfo*/);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CompactionContext createCompactionContext() {
|
||||
public CompactionContext createCompaction() {
|
||||
return new DefaultCompactionContext();
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -113,7 +113,6 @@ public class HStore implements Store {
|
|||
// This stores directory in the filesystem.
|
||||
private final HRegion region;
|
||||
private final HColumnDescriptor family;
|
||||
final CompactionPolicy compactionPolicy;
|
||||
private final HRegionFileSystem fs;
|
||||
private final Configuration conf;
|
||||
private final CacheConfig cacheConf;
|
||||
|
|
@ -129,7 +128,6 @@ public class HStore implements Store {
|
|||
|
||||
private ScanInfo scanInfo;
|
||||
|
||||
private final StoreFileManager storeFileManager;
|
||||
final List<StoreFile> filesCompacting = Lists.newArrayList();
|
||||
|
||||
// All access must be synchronized.
|
||||
|
|
@ -210,12 +208,8 @@ public class HStore implements Store {
|
|||
"hbase.hstore.close.check.interval", 10*1000*1000 /* 10 MB */);
|
||||
}
|
||||
|
||||
storeEngine = StoreEngine.create(this, this.conf, this.comparator);
|
||||
// Copy some things to local fields for convenience.
|
||||
this.storeFileManager = storeEngine.getStoreFileManager();
|
||||
this.compactionPolicy = storeEngine.getCompactionPolicy();
|
||||
|
||||
this.storeFileManager.loadFiles(loadStoreFiles());
|
||||
this.storeEngine = StoreEngine.create(this, this.conf, this.comparator);
|
||||
this.storeEngine.getStoreFileManager().loadFiles(loadStoreFiles());
|
||||
|
||||
// Initialize checksum type from name. The names are CRC32, CRC32C, etc.
|
||||
this.checksumType = getChecksumType(conf);
|
||||
|
|
@ -479,7 +473,7 @@ public class HStore implements Store {
|
|||
*/
|
||||
@Override
|
||||
public Collection<StoreFile> getStorefiles() {
|
||||
return this.storeFileManager.getStorefiles();
|
||||
return this.storeEngine.getStoreFileManager().getStorefiles();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -560,7 +554,7 @@ public class HStore implements Store {
|
|||
// Append the new storefile into the list
|
||||
this.lock.writeLock().lock();
|
||||
try {
|
||||
this.storeFileManager.insertNewFile(sf);
|
||||
this.storeEngine.getStoreFileManager().insertNewFile(sf);
|
||||
} finally {
|
||||
// We need the lock, as long as we are updating the storeFiles
|
||||
// or changing the memstore. Let us release it before calling
|
||||
|
|
@ -579,7 +573,7 @@ public class HStore implements Store {
|
|||
this.lock.writeLock().lock();
|
||||
try {
|
||||
// Clear so metrics doesn't find them.
|
||||
ImmutableCollection<StoreFile> result = storeFileManager.clearFiles();
|
||||
ImmutableCollection<StoreFile> result = storeEngine.getStoreFileManager().clearFiles();
|
||||
|
||||
if (!result.isEmpty()) {
|
||||
// initialize the thread pool for closing store files in parallel.
|
||||
|
|
@ -879,7 +873,7 @@ public class HStore implements Store {
|
|||
throws IOException {
|
||||
this.lock.writeLock().lock();
|
||||
try {
|
||||
this.storeFileManager.insertNewFile(sf);
|
||||
this.storeEngine.getStoreFileManager().insertNewFile(sf);
|
||||
this.memstore.clearSnapshot(set);
|
||||
} finally {
|
||||
// We need the lock, as long as we are updating the storeFiles
|
||||
|
|
@ -919,7 +913,8 @@ public class HStore implements Store {
|
|||
List<KeyValueScanner> memStoreScanners;
|
||||
this.lock.readLock().lock();
|
||||
try {
|
||||
storeFilesToScan = this.storeFileManager.getFilesForScanOrGet(isGet, startRow, stopRow);
|
||||
storeFilesToScan =
|
||||
this.storeEngine.getStoreFileManager().getFilesForScanOrGet(isGet, startRow, stopRow);
|
||||
memStoreScanners = this.memstore.getScanners();
|
||||
} finally {
|
||||
this.lock.readLock().unlock();
|
||||
|
|
@ -1083,7 +1078,7 @@ public class HStore implements Store {
|
|||
this.lock.readLock().lock();
|
||||
try {
|
||||
synchronized (filesCompacting) {
|
||||
filesToCompact = Lists.newArrayList(storeFileManager.getStorefiles());
|
||||
filesToCompact = Lists.newArrayList(storeEngine.getStoreFileManager().getStorefiles());
|
||||
if (!filesCompacting.isEmpty()) {
|
||||
// exclude all files older than the newest file we're currently
|
||||
// compacting. this allows us to preserve contiguity (HBASE-2856)
|
||||
|
|
@ -1098,7 +1093,7 @@ public class HStore implements Store {
|
|||
}
|
||||
|
||||
filesToCompact = filesToCompact.subList(count - N, count);
|
||||
isMajor = (filesToCompact.size() == storeFileManager.getStorefileCount());
|
||||
isMajor = (filesToCompact.size() == storeEngine.getStoreFileManager().getStorefileCount());
|
||||
filesCompacting.addAll(filesToCompact);
|
||||
Collections.sort(filesCompacting, StoreFile.Comparators.SEQ_ID);
|
||||
}
|
||||
|
|
@ -1129,7 +1124,7 @@ public class HStore implements Store {
|
|||
|
||||
@Override
|
||||
public boolean hasReferences() {
|
||||
return StoreUtils.hasReferences(this.storeFileManager.getStorefiles());
|
||||
return StoreUtils.hasReferences(this.storeEngine.getStoreFileManager().getStorefiles());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -1139,14 +1134,15 @@ public class HStore implements Store {
|
|||
|
||||
@Override
|
||||
public boolean isMajorCompaction() throws IOException {
|
||||
for (StoreFile sf : this.storeFileManager.getStorefiles()) {
|
||||
for (StoreFile sf : this.storeEngine.getStoreFileManager().getStorefiles()) {
|
||||
// TODO: what are these reader checks all over the place?
|
||||
if (sf.getReader() == null) {
|
||||
LOG.debug("StoreFile " + sf + " has null Reader");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return compactionPolicy.isMajorCompaction(this.storeFileManager.getStorefiles());
|
||||
return storeEngine.getCompactionPolicy().isMajorCompaction(
|
||||
this.storeEngine.getStoreFileManager().getStorefiles());
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -1307,7 +1303,7 @@ public class HStore implements Store {
|
|||
// delete old store files until we have sent out notification of
|
||||
// change in case old files are still being accessed by outstanding
|
||||
// scanners.
|
||||
this.storeFileManager.addCompactionResults(compactedFiles, result);
|
||||
this.storeEngine.getStoreFileManager().addCompactionResults(compactedFiles, result);
|
||||
filesCompacting.removeAll(compactedFiles); // safe bc: lock.writeLock()
|
||||
} finally {
|
||||
// We need the lock, as long as we are updating the storeFiles
|
||||
|
|
@ -1336,7 +1332,7 @@ public class HStore implements Store {
|
|||
// 4. Compute new store size
|
||||
this.storeSize = 0L;
|
||||
this.totalUncompressedBytes = 0L;
|
||||
for (StoreFile hsf : this.storeFileManager.getStorefiles()) {
|
||||
for (StoreFile hsf : this.storeEngine.getStoreFileManager().getStorefiles()) {
|
||||
StoreFile.Reader r = hsf.getReader();
|
||||
if (r == null) {
|
||||
LOG.warn("StoreFile " + hsf + " has a null Reader");
|
||||
|
|
@ -1384,16 +1380,16 @@ public class HStore implements Store {
|
|||
this.memstore.getRowKeyAtOrBefore(state);
|
||||
// Check if match, if we got a candidate on the asked for 'kv' row.
|
||||
// Process each relevant store file. Run through from newest to oldest.
|
||||
Iterator<StoreFile> sfIterator =
|
||||
this.storeFileManager.getCandidateFilesForRowKeyBefore(state.getTargetKey());
|
||||
Iterator<StoreFile> sfIterator = this.storeEngine.getStoreFileManager()
|
||||
.getCandidateFilesForRowKeyBefore(state.getTargetKey());
|
||||
while (sfIterator.hasNext()) {
|
||||
StoreFile sf = sfIterator.next();
|
||||
sfIterator.remove(); // Remove sf from iterator.
|
||||
boolean haveNewCandidate = rowAtOrBeforeFromStoreFile(sf, state);
|
||||
if (haveNewCandidate) {
|
||||
// TODO: we may have an optimization here which stops the search if we find exact match.
|
||||
sfIterator = this.storeFileManager.updateCandidateFilesForRowKeyBefore(sfIterator,
|
||||
state.getTargetKey(), state.getCandidate());
|
||||
sfIterator = this.storeEngine.getStoreFileManager().updateCandidateFilesForRowKeyBefore(
|
||||
sfIterator, state.getTargetKey(), state.getCandidate());
|
||||
}
|
||||
}
|
||||
return state.getCandidate();
|
||||
|
|
@ -1537,7 +1533,7 @@ public class HStore implements Store {
|
|||
assert false : "getSplitPoint() called on a region that can't split!";
|
||||
return null;
|
||||
}
|
||||
return this.storeFileManager.getSplitPoint();
|
||||
return this.storeEngine.getStoreFileManager().getSplitPoint();
|
||||
} catch(IOException e) {
|
||||
LOG.warn("Failed getting store size for " + this, e);
|
||||
} finally {
|
||||
|
|
@ -1594,7 +1590,7 @@ public class HStore implements Store {
|
|||
@Override
|
||||
// TODO: why is there this and also getNumberOfStorefiles?! Remove one.
|
||||
public int getStorefilesCount() {
|
||||
return this.storeFileManager.getStorefileCount();
|
||||
return this.storeEngine.getStoreFileManager().getStorefileCount();
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -1605,7 +1601,7 @@ public class HStore implements Store {
|
|||
@Override
|
||||
public long getStorefilesSize() {
|
||||
long size = 0;
|
||||
for (StoreFile s: this.storeFileManager.getStorefiles()) {
|
||||
for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
|
||||
StoreFile.Reader r = s.getReader();
|
||||
if (r == null) {
|
||||
LOG.warn("StoreFile " + s + " has a null Reader");
|
||||
|
|
@ -1619,7 +1615,7 @@ public class HStore implements Store {
|
|||
@Override
|
||||
public long getStorefilesIndexSize() {
|
||||
long size = 0;
|
||||
for (StoreFile s: this.storeFileManager.getStorefiles()) {
|
||||
for (StoreFile s: this.storeEngine.getStoreFileManager().getStorefiles()) {
|
||||
StoreFile.Reader r = s.getReader();
|
||||
if (r == null) {
|
||||
LOG.warn("StoreFile " + s + " has a null Reader");
|
||||
|
|
@ -1633,7 +1629,7 @@ public class HStore implements Store {
|
|||
@Override
|
||||
public long getTotalStaticIndexSize() {
|
||||
long size = 0;
|
||||
for (StoreFile s : this.storeFileManager.getStorefiles()) {
|
||||
for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
|
||||
size += s.getReader().getUncompressedDataIndexSize();
|
||||
}
|
||||
return size;
|
||||
|
|
@ -1642,7 +1638,7 @@ public class HStore implements Store {
|
|||
@Override
|
||||
public long getTotalStaticBloomSize() {
|
||||
long size = 0;
|
||||
for (StoreFile s : this.storeFileManager.getStorefiles()) {
|
||||
for (StoreFile s : this.storeEngine.getStoreFileManager().getStorefiles()) {
|
||||
StoreFile.Reader r = s.getReader();
|
||||
size += r.getTotalBloomSize();
|
||||
}
|
||||
|
|
@ -1656,12 +1652,12 @@ public class HStore implements Store {
|
|||
|
||||
@Override
|
||||
public int getCompactPriority() {
|
||||
return this.storeFileManager.getStoreCompactionPriority();
|
||||
return this.storeEngine.getStoreFileManager().getStoreCompactionPriority();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean throttleCompaction(long compactionSize) {
|
||||
return compactionPolicy.throttleCompaction(compactionSize);
|
||||
return storeEngine.getCompactionPolicy().throttleCompaction(compactionSize);
|
||||
}
|
||||
|
||||
public HRegion getHRegion() {
|
||||
|
|
@ -1780,7 +1776,8 @@ public class HStore implements Store {
|
|||
|
||||
@Override
|
||||
public boolean needsCompaction() {
|
||||
return compactionPolicy.needsCompaction(this.storeFileManager.getStorefiles(), filesCompacting);
|
||||
return storeEngine.getCompactionPolicy().needsCompaction(
|
||||
this.storeEngine.getStoreFileManager().getStorefiles(), filesCompacting);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -1789,7 +1786,7 @@ public class HStore implements Store {
|
|||
}
|
||||
|
||||
public static final long FIXED_OVERHEAD =
|
||||
ClassSize.align((19 * ClassSize.REFERENCE) + (4 * Bytes.SIZEOF_LONG)
|
||||
ClassSize.align((17 * ClassSize.REFERENCE) + (4 * Bytes.SIZEOF_LONG)
|
||||
+ (2 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_BOOLEAN);
|
||||
|
||||
public static final long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD
|
||||
|
|
|
|||
|
|
@ -39,14 +39,9 @@ import org.apache.hadoop.hbase.util.ReflectionUtils;
|
|||
@InterfaceAudience.Private
|
||||
public abstract class StoreEngine<
|
||||
CP extends CompactionPolicy, C extends Compactor, SFM extends StoreFileManager> {
|
||||
protected final Store store;
|
||||
protected final Configuration conf;
|
||||
protected final KVComparator comparator;
|
||||
|
||||
protected CP compactionPolicy;
|
||||
protected C compactor;
|
||||
protected SFM storeFileManager;
|
||||
private boolean isInitialized = false;
|
||||
|
||||
/**
|
||||
* The name of the configuration parameter that specifies the class of
|
||||
|
|
@ -61,7 +56,6 @@ public abstract class StoreEngine<
|
|||
* @return Compaction policy to use.
|
||||
*/
|
||||
public CompactionPolicy getCompactionPolicy() {
|
||||
createComponentsOnce();
|
||||
return this.compactionPolicy;
|
||||
}
|
||||
|
||||
|
|
@ -69,7 +63,6 @@ public abstract class StoreEngine<
|
|||
* @return Compactor to use.
|
||||
*/
|
||||
public Compactor getCompactor() {
|
||||
createComponentsOnce();
|
||||
return this.compactor;
|
||||
}
|
||||
|
||||
|
|
@ -77,33 +70,27 @@ public abstract class StoreEngine<
|
|||
* @return Store file manager to use.
|
||||
*/
|
||||
public StoreFileManager getStoreFileManager() {
|
||||
createComponentsOnce();
|
||||
return this.storeFileManager;
|
||||
}
|
||||
|
||||
protected StoreEngine(Configuration conf, Store store, KVComparator comparator) {
|
||||
this.store = store;
|
||||
this.conf = conf;
|
||||
this.comparator = comparator;
|
||||
}
|
||||
|
||||
public CompactionContext createCompaction() {
|
||||
createComponentsOnce();
|
||||
return this.createCompactionContext();
|
||||
}
|
||||
|
||||
protected abstract CompactionContext createCompactionContext();
|
||||
/**
|
||||
* Creates an instance of a compaction context specific to this engine.
|
||||
* Doesn't actually select or start a compaction. See CompactionContext class comment.
|
||||
* @return New CompactionContext object.
|
||||
*/
|
||||
public abstract CompactionContext createCompaction() throws IOException;
|
||||
|
||||
/**
|
||||
* Create the StoreEngine's components.
|
||||
*/
|
||||
protected abstract void createComponents();
|
||||
protected abstract void createComponents(
|
||||
Configuration conf, Store store, KVComparator kvComparator) throws IOException;
|
||||
|
||||
private void createComponentsOnce() {
|
||||
if (isInitialized) return;
|
||||
createComponents();
|
||||
private void createComponentsOnce(
|
||||
Configuration conf, Store store, KVComparator kvComparator) throws IOException {
|
||||
assert compactor == null && compactionPolicy == null && storeFileManager == null;
|
||||
createComponents(conf, store, kvComparator);
|
||||
assert compactor != null && compactionPolicy != null && storeFileManager != null;
|
||||
isInitialized = true;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -114,13 +101,14 @@ public abstract class StoreEngine<
|
|||
* @param kvComparator KVComparator for storeFileManager.
|
||||
* @return StoreEngine to use.
|
||||
*/
|
||||
public static StoreEngine create(Store store, Configuration conf, KVComparator kvComparator)
|
||||
throws IOException {
|
||||
public static StoreEngine<?, ?, ?> create(
|
||||
Store store, Configuration conf, KVComparator kvComparator) throws IOException {
|
||||
String className = conf.get(STORE_ENGINE_CLASS_KEY, DEFAULT_STORE_ENGINE_CLASS.getName());
|
||||
try {
|
||||
return ReflectionUtils.instantiateWithCustomCtor(className,
|
||||
new Class[] { Configuration.class, Store.class, KVComparator.class },
|
||||
new Object[] { conf, store, kvComparator });
|
||||
StoreEngine<?,?,?> se = ReflectionUtils.instantiateWithCustomCtor(
|
||||
className, new Class[] { }, new Object[] { });
|
||||
se.createComponentsOnce(conf, store, kvComparator);
|
||||
return se;
|
||||
} catch (Exception e) {
|
||||
throw new IOException("Unable to load configured store engine '" + className + "'", e);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -27,7 +27,8 @@ import org.apache.hadoop.hbase.regionserver.StoreFile;
|
|||
|
||||
|
||||
/**
|
||||
* This class holds all "physical" details necessary to run a compaction.
|
||||
* This class holds all "physical" details necessary to run a compaction,
|
||||
* and abstracts away the details specific to a particular compaction.
|
||||
* It also has compaction request with all the logical details.
|
||||
* Hence, this class is basically the compaction.
|
||||
*/
|
||||
|
|
|
|||
|
|
@ -309,7 +309,7 @@ public class TestCompaction extends HBaseTestCase {
|
|||
conf.setFloat("hbase.hregion.majorcompaction.jitter", jitterPct);
|
||||
|
||||
HStore s = ((HStore) r.getStore(COLUMN_FAMILY));
|
||||
s.compactionPolicy.setConf(conf);
|
||||
s.storeEngine.getCompactionPolicy().setConf(conf);
|
||||
try {
|
||||
createStoreFile(r);
|
||||
createStoreFile(r);
|
||||
|
|
@ -321,7 +321,7 @@ public class TestCompaction extends HBaseTestCase {
|
|||
assertEquals(2, s.getStorefilesCount());
|
||||
|
||||
// ensure that major compaction time is deterministic
|
||||
DefaultCompactionPolicy c = (DefaultCompactionPolicy)s.compactionPolicy;
|
||||
DefaultCompactionPolicy c = (DefaultCompactionPolicy)s.storeEngine.getCompactionPolicy();
|
||||
Collection<StoreFile> storeFiles = s.getStorefiles();
|
||||
long mcTime = c.getNextMajorCompactTime(storeFiles);
|
||||
for (int i = 0; i < 10; ++i) {
|
||||
|
|
|
|||
|
|
@ -234,8 +234,8 @@ public class TestDefaultCompactSelection extends TestCase {
|
|||
throws IOException {
|
||||
store.forceMajor = forcemajor;
|
||||
//Test Default compactions
|
||||
CompactionRequest result = ((DefaultCompactionPolicy)store.compactionPolicy).selectCompaction(
|
||||
candidates, new ArrayList<StoreFile>(), false, isOffPeak, forcemajor);
|
||||
CompactionRequest result = ((DefaultCompactionPolicy)store.storeEngine.getCompactionPolicy())
|
||||
.selectCompaction(candidates, new ArrayList<StoreFile>(), false, isOffPeak, forcemajor);
|
||||
List<StoreFile> actual = new ArrayList<StoreFile>(result.getFiles());
|
||||
if (isOffPeak && !forcemajor) {
|
||||
assertTrue(result.isOffPeak());
|
||||
|
|
@ -288,7 +288,7 @@ public class TestDefaultCompactSelection extends TestCase {
|
|||
compactEquals(sfCreate(100,50,23,12,12), true, 23, 12, 12);
|
||||
conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 1);
|
||||
conf.setFloat("hbase.hregion.majorcompaction.jitter", 0);
|
||||
store.compactionPolicy.setConf(conf);
|
||||
store.storeEngine.getCompactionPolicy().setConf(conf);
|
||||
try {
|
||||
// trigger an aged major compaction
|
||||
compactEquals(sfCreate(50,25,12,12), 50, 25, 12, 12);
|
||||
|
|
@ -321,7 +321,7 @@ public class TestDefaultCompactSelection extends TestCase {
|
|||
*/
|
||||
// set an off-peak compaction threshold
|
||||
this.conf.setFloat("hbase.hstore.compaction.ratio.offpeak", 5.0F);
|
||||
store.compactionPolicy.setConf(this.conf);
|
||||
store.storeEngine.getCompactionPolicy().setConf(this.conf);
|
||||
// Test with and without the flag.
|
||||
compactEquals(sfCreate(999, 50, 12, 12, 1), false, true, 50, 12, 12, 1);
|
||||
compactEquals(sfCreate(999, 50, 12, 12, 1), 12, 12, 1);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,59 @@
|
|||
/*
|
||||
*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hadoop.hbase.regionserver;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.hbase.HBaseConfiguration;
|
||||
import org.apache.hadoop.hbase.KeyValue.KVComparator;
|
||||
import org.apache.hadoop.hbase.SmallTests;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactionPolicy;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
@Category(SmallTests.class)
|
||||
public class TestDefaultStoreEngine {
|
||||
public static class DummyCompactor extends DefaultCompactor {
|
||||
public DummyCompactor(Configuration conf, Store store) {
|
||||
super(conf, store);
|
||||
}
|
||||
}
|
||||
|
||||
public static class DummyCompactionPolicy extends DefaultCompactionPolicy {
|
||||
public DummyCompactionPolicy(Configuration conf, StoreConfigInformation storeConfigInfo) {
|
||||
super(conf, storeConfigInfo);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCustomPolicyAndCompactor() throws Exception {
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
conf.set(DefaultStoreEngine.DEFAULT_COMPACTOR_CLASS_KEY, DummyCompactor.class.getName());
|
||||
conf.set(DefaultStoreEngine.DEFAULT_COMPACTION_POLICY_CLASS_KEY,
|
||||
DummyCompactionPolicy.class.getName());
|
||||
Store mockStore = Mockito.mock(Store.class);
|
||||
StoreEngine<?, ?, ?> se = StoreEngine.create(mockStore, conf, new KVComparator());
|
||||
Assert.assertTrue(se instanceof DefaultStoreEngine);
|
||||
Assert.assertTrue(se.getCompactionPolicy() instanceof DummyCompactionPolicy);
|
||||
Assert.assertTrue(se.getCompactor() instanceof DummyCompactor);
|
||||
}
|
||||
}
|
||||
|
|
@ -49,6 +49,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
|
|||
import org.apache.hadoop.hbase.HRegionInfo;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.KeyValue;
|
||||
import org.apache.hadoop.hbase.KeyValue.KVComparator;
|
||||
import org.apache.hadoop.hbase.MediumTests;
|
||||
import org.apache.hadoop.hbase.client.Get;
|
||||
import org.apache.hadoop.hbase.io.compress.Compression;
|
||||
|
|
@ -59,6 +60,7 @@ import org.apache.hadoop.hbase.monitoring.MonitoredTask;
|
|||
import org.apache.hadoop.hbase.regionserver.StoreFileInfo;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionContext;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.DefaultCompactor;
|
||||
import org.apache.hadoop.hbase.regionserver.wal.HLog;
|
||||
import org.apache.hadoop.hbase.regionserver.wal.HLogFactory;
|
||||
import org.apache.hadoop.hbase.security.User;
|
||||
|
|
@ -853,5 +855,22 @@ public class TestStore extends TestCase {
|
|||
assertTrue(store.throttleCompaction(anyValue + 1));
|
||||
assertFalse(store.throttleCompaction(anyValue));
|
||||
}
|
||||
|
||||
public static class DummyStoreEngine extends DefaultStoreEngine {
|
||||
public static DefaultCompactor lastCreatedCompactor = null;
|
||||
@Override
|
||||
protected void createComponents(
|
||||
Configuration conf, Store store, KVComparator comparator) throws IOException {
|
||||
super.createComponents(conf, store, comparator);
|
||||
lastCreatedCompactor = this.compactor;
|
||||
}
|
||||
}
|
||||
|
||||
public void testStoreUsesSearchEngineOverride() throws Exception {
|
||||
Configuration conf = HBaseConfiguration.create();
|
||||
conf.set(StoreEngine.STORE_ENGINE_CLASS_KEY, DummyStoreEngine.class.getName());
|
||||
init(this.getName(), conf);
|
||||
assertEquals(DummyStoreEngine.lastCreatedCompactor, this.store.storeEngine.getCompactor());
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue