HBASE-8870 Store.needsCompaction() should include minFilesToCompact (Liang Xie)
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1526320 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
22b77dc791
commit
a21deb20f1
|
@ -392,6 +392,6 @@ public class RatioBasedCompactionPolicy extends CompactionPolicy {
|
|||
public boolean needsCompaction(final Collection<StoreFile> storeFiles,
|
||||
final List<StoreFile> filesCompacting) {
|
||||
int numCandidates = storeFiles.size() - filesCompacting.size();
|
||||
return numCandidates > comConf.getMinFilesToCompact();
|
||||
return numCandidates >= comConf.getMinFilesToCompact();
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1165,6 +1165,30 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
|
|||
return new HTable(getConfiguration(), tableName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a table.
|
||||
* @param htd
|
||||
* @param families
|
||||
* @param c Configuration to use
|
||||
* @return An HTable instance for the created table.
|
||||
* @throws IOException
|
||||
*/
|
||||
public HTable createTable(HTableDescriptor htd, byte[][] families, Configuration c)
|
||||
throws IOException {
|
||||
for(byte[] family : families) {
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||
// Disable blooms (they are on by default as of 0.95) but we disable them here because
|
||||
// tests have hard coded counts of what to expect in block cache, etc., and blooms being
|
||||
// on is interfering.
|
||||
hcd.setBloomFilterType(BloomType.NONE);
|
||||
htd.addFamily(hcd);
|
||||
}
|
||||
getHBaseAdmin().createTable(htd);
|
||||
// HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
|
||||
waitUntilAllRegionsAssigned(htd.getTableName());
|
||||
return new HTable(c, htd.getTableName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a table.
|
||||
* @param tableName
|
||||
|
@ -1176,19 +1200,7 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility {
|
|||
public HTable createTable(TableName tableName, byte[][] families,
|
||||
final Configuration c)
|
||||
throws IOException {
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
for(byte[] family : families) {
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||
// Disable blooms (they are on by default as of 0.95) but we disable them here because
|
||||
// tests have hard coded counts of what to expect in block cache, etc., and blooms being
|
||||
// on is interfering.
|
||||
hcd.setBloomFilterType(BloomType.NONE);
|
||||
desc.addFamily(hcd);
|
||||
}
|
||||
getHBaseAdmin().createTable(desc);
|
||||
// HBaseAdmin only waits for regions to appear in hbase:meta we should wait until they are assigned
|
||||
waitUntilAllRegionsAssigned(tableName);
|
||||
return new HTable(c, tableName);
|
||||
return createTable(new HTableDescriptor(tableName), families, c);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -55,9 +55,7 @@ import org.apache.hadoop.hbase.regionserver.KeyValueScanner;
|
|||
import org.apache.hadoop.hbase.regionserver.RegionCoprocessorHost;
|
||||
import org.apache.hadoop.hbase.regionserver.ScanType;
|
||||
import org.apache.hadoop.hbase.regionserver.Store;
|
||||
import org.apache.hadoop.hbase.regionserver.StoreFile;
|
||||
import org.apache.hadoop.hbase.regionserver.StoreScanner;
|
||||
import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
@ -249,14 +247,8 @@ public class TestRegionObserverScannerOpenHook {
|
|||
admin.flush(region.getRegionName());
|
||||
|
||||
// run a compaction, which normally would should get rid of the data
|
||||
Store s = region.getStores().get(A);
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
WaitableCompactionRequest request = new WaitableCompactionRequest(s.getStorefiles(), latch);
|
||||
rs.compactSplitThread.requestCompaction(region, s,
|
||||
"compact for testRegionObserverCompactionTimeStacking", Store.PRIORITY_USER, request);
|
||||
// wait for the compaction to complete
|
||||
latch.await();
|
||||
|
||||
// wait for the compaction checker to complete
|
||||
Thread.sleep(1000);
|
||||
// check both rows to ensure that they aren't there
|
||||
Get get = new Get(ROW);
|
||||
Result r = table.get(get);
|
||||
|
@ -273,26 +265,4 @@ public class TestRegionObserverScannerOpenHook {
|
|||
table.close();
|
||||
UTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
/**
|
||||
* A simple compaction on which you can wait for the passed in latch until the compaction finishes
|
||||
* (either successfully or if it failed).
|
||||
*/
|
||||
public static class WaitableCompactionRequest extends CompactionRequest {
|
||||
private CountDownLatch done;
|
||||
|
||||
/**
|
||||
* Constructor for a custom compaction. Uses the setXXX methods to update the state of the
|
||||
* compaction before being used.
|
||||
*/
|
||||
public WaitableCompactionRequest(Collection<StoreFile> files, CountDownLatch finished) {
|
||||
super(files);
|
||||
this.done = finished;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void afterExecute() {
|
||||
this.done.countDown();
|
||||
}
|
||||
}
|
||||
}
|
|
@ -33,6 +33,7 @@ import org.apache.hadoop.conf.Configuration;
|
|||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.FileSystem;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
|
@ -280,11 +281,19 @@ public class TestSnapshotFromMaster {
|
|||
HBaseAdmin admin = UTIL.getHBaseAdmin();
|
||||
// make sure we don't fail on listing snapshots
|
||||
SnapshotTestingUtils.assertNoSnapshots(admin);
|
||||
|
||||
// recreate test table with disabled compactions; otherwise compaction may happen before
|
||||
// snapshot, the call after snapshot will be a no-op and checks will fail
|
||||
UTIL.deleteTable(TABLE_NAME);
|
||||
HTableDescriptor htd = new HTableDescriptor(TABLE_NAME);
|
||||
htd.setCompactionEnabled(false);
|
||||
UTIL.createTable(htd, new byte[][] { TEST_FAM }, UTIL.getConfiguration());
|
||||
// load the table (creates 4 hfiles)
|
||||
UTIL.loadTable(new HTable(UTIL.getConfiguration(), TABLE_NAME), TEST_FAM);
|
||||
|
||||
// disable the table so we can take a snapshot
|
||||
admin.disableTable(TABLE_NAME);
|
||||
htd.setCompactionEnabled(true);
|
||||
|
||||
// take a snapshot of the table
|
||||
String snapshotName = "snapshot";
|
||||
|
@ -298,6 +307,9 @@ public class TestSnapshotFromMaster {
|
|||
// ensure we only have one snapshot
|
||||
SnapshotTestingUtils.assertOneSnapshotThatMatches(admin, snapshotNameBytes, TABLE_NAME);
|
||||
|
||||
// enable compactions now
|
||||
admin.modifyTable(TABLE_NAME, htd);
|
||||
|
||||
// renable the table so we can compact the regions
|
||||
admin.enableTable(TABLE_NAME);
|
||||
|
||||
|
|
Loading…
Reference in New Issue