HBASE-14567 Tuneup hanging test TestMobCompactor and TestMobSweeper; ADDENDUM (jingcheng du)
This commit is contained in:
stack
parent
e37e72c160
commit
587f5bc11f
|
|
@ -68,8 +68,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig;
|
|||
import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||
import org.apache.hadoop.hbase.mob.MobConstants;
|
||||
import org.apache.hadoop.hbase.mob.MobUtils;
|
||||
import org.apache.hadoop.hbase.mob.compactions.MobCompactor;
|
||||
import org.apache.hadoop.hbase.mob.compactions.PartitionedMobCompactor;
|
||||
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoResponse.CompactionState;
|
||||
import org.apache.hadoop.hbase.regionserver.BloomType;
|
||||
import org.apache.hadoop.hbase.regionserver.HRegion;
|
||||
|
|
@ -81,10 +79,8 @@ import org.apache.hadoop.hbase.testclassification.LargeTests;
|
|||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
|
||||
import org.apache.hadoop.hbase.util.Threads;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
|
@ -92,17 +88,16 @@ import org.junit.experimental.categories.Category;
|
|||
@Category(LargeTests.class)
|
||||
public class TestMobCompactor {
|
||||
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
private Configuration conf = null;
|
||||
private String tableNameAsString;
|
||||
private static Configuration conf = null;
|
||||
private TableName tableName;
|
||||
private static Connection conn;
|
||||
private BufferedMutator bufMut;
|
||||
private Table hTable;
|
||||
private Admin admin;
|
||||
private Table table;
|
||||
private static Admin admin;
|
||||
private HTableDescriptor desc;
|
||||
private HColumnDescriptor hcd1;
|
||||
private HColumnDescriptor hcd2;
|
||||
private FileSystem fs;
|
||||
private static FileSystem fs;
|
||||
private static final String family1 = "family1";
|
||||
private static final String family2 = "family2";
|
||||
private static final String qf1 = "qualifier1";
|
||||
|
|
@ -127,6 +122,9 @@ public class TestMobCompactor {
|
|||
TEST_UTIL.startMiniCluster(1);
|
||||
pool = createThreadPool(TEST_UTIL.getConfiguration());
|
||||
conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration(), pool);
|
||||
fs = TEST_UTIL.getTestFileSystem();
|
||||
conf = TEST_UTIL.getConfiguration();
|
||||
admin = TEST_UTIL.getHBaseAdmin();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
|
|
@ -136,249 +134,128 @@ public class TestMobCompactor {
|
|||
TEST_UTIL.shutdownMiniCluster();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
fs = TEST_UTIL.getTestFileSystem();
|
||||
conf = TEST_UTIL.getConfiguration();
|
||||
long tid = System.currentTimeMillis();
|
||||
tableNameAsString = "testMob" + tid;
|
||||
public void setUp(String tableNameAsString) throws IOException {
|
||||
tableName = TableName.valueOf(tableNameAsString);
|
||||
hcd1 = new HColumnDescriptor(family1);
|
||||
hcd1.setMobEnabled(true);
|
||||
hcd1.setMobThreshold(5);
|
||||
hcd1.setMaxVersions(4);
|
||||
hcd2 = new HColumnDescriptor(family2);
|
||||
hcd2.setMobEnabled(true);
|
||||
hcd2.setMobThreshold(5);
|
||||
hcd2.setMaxVersions(4);
|
||||
desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(hcd1);
|
||||
desc.addFamily(hcd2);
|
||||
admin = TEST_UTIL.getHBaseAdmin();
|
||||
admin.createTable(desc, getSplitKeys());
|
||||
hTable = conn.getTable(tableName);
|
||||
table = conn.getTable(tableName);
|
||||
bufMut = conn.getBufferedMutator(tableName);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
admin.close();
|
||||
hTable.close();
|
||||
fs.delete(TEST_UTIL.getDataTestDir(), true);
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionWithoutDelFiles() throws Exception {
|
||||
resetConf();
|
||||
// create a table with namespace
|
||||
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
|
||||
String tableNameAsString = "ns:testCompactionWithoutDelFiles";
|
||||
admin.createNamespace(namespaceDescriptor);
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
SecureRandom rng = new SecureRandom();
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
rng.nextBytes(keyBytes);
|
||||
String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key cfKey = new SecretKeySpec(keyBytes, algorithm);
|
||||
byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
|
||||
conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
|
||||
TableName tableName = TableName.valueOf(tableNameAsString);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(family1);
|
||||
hcd.setMobEnabled(true);
|
||||
hcd.setMobThreshold(0);
|
||||
hcd.setMaxVersions(4);
|
||||
hcd.setEncryptionType(algorithm);
|
||||
hcd.setEncryptionKey(encryptionKey);
|
||||
HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
|
||||
hcd2.setMobEnabled(true);
|
||||
hcd2.setMobThreshold(0);
|
||||
hcd2.setMaxVersions(4);
|
||||
desc.addFamily(hcd);
|
||||
desc.addFamily(hcd2);
|
||||
admin.createTable(desc, getSplitKeys());
|
||||
Table table = conn.getTable(tableName);
|
||||
BufferedMutator bufMut = conn.getBufferedMutator(tableName);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("Before compaction: mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
|
||||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("After compaction: mob file count", regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
Assert.assertTrue(verifyEncryption(tableName, family1));
|
||||
table.close();
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
admin.deleteNamespace("ns");
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionWithDelFiles() throws Exception {
|
||||
resetConf();
|
||||
int batchSize = 2;
|
||||
conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
|
||||
assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
|
||||
countMobCells(hTable));
|
||||
assertEquals("Before deleting: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before deleting: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
|
||||
createDelFile();
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("Before compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
|
||||
// do the mob file compaction
|
||||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize),
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After compaction: family1 del file count", 0,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
assertRefFileNameEqual(family1);
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
public void testMinorCompactionWithDelFiles() throws Exception {
|
||||
public void testMinorCompaction() throws Exception {
|
||||
resetConf();
|
||||
int mergeSize = 5000;
|
||||
// change the mob compaction merge size
|
||||
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
|
||||
|
||||
// create a table with namespace
|
||||
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
|
||||
String tableNameAsString = "ns:testMinorCompaction";
|
||||
admin.createNamespace(namespaceDescriptor);
|
||||
setUp(tableNameAsString);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
|
||||
countMobCells(hTable));
|
||||
assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
|
||||
countMobCells(table));
|
||||
assertEquals("Before deleting: mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
|
||||
int largeFilesCount = countLargeFiles(mergeSize, family1);
|
||||
createDelFile();
|
||||
int largeFilesCount = countLargeFiles(mergeSize, tableName, family1);
|
||||
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
assertEquals("Before compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("Before compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
countFiles(tableName, false, family2));
|
||||
|
||||
// do the mob file compaction
|
||||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
assertEquals("After compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
// After the compaction, the files smaller than the mob compaction merge size
|
||||
// is merge to one file
|
||||
assertEquals("After compaction: family1 mob file count", largeFilesCount + regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
countFiles(tableName, false, family2));
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionWithHFileLink() throws IOException, InterruptedException {
|
||||
resetConf();
|
||||
String tableNameAsString = "testCompactionWithHFileLink";
|
||||
setUp(tableNameAsString);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
long tid = System.currentTimeMillis();
|
||||
byte[] snapshotName1 = Bytes.toBytes("snaptb-" + tid);
|
||||
// take a snapshot
|
||||
admin.snapshot(snapshotName1, tableName);
|
||||
|
||||
createDelFile();
|
||||
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
assertEquals("Before compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("Before compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
countFiles(tableName, false, family2));
|
||||
|
||||
// do the mob compaction
|
||||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After first compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("After first compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("After first compaction: mob rows count", regionNum
|
||||
* (rowNumPerRegion - delRowNum), countMobRows(table));
|
||||
assertEquals("After first compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("After first compaction: family1 mob file count", regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After first compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After first compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After first compaction: family1 del file count", 0,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After first compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
countFiles(tableName, false, family2));
|
||||
assertEquals("After first compaction: family1 hfilelink count", 0, countHFileLinks(family1));
|
||||
assertEquals("After first compaction: family2 hfilelink count", 0, countHFileLinks(family2));
|
||||
|
||||
|
|
@ -387,33 +264,32 @@ public class TestMobCompactor {
|
|||
admin.restoreSnapshot(snapshotName1);
|
||||
admin.enableTable(tableName);
|
||||
|
||||
assertEquals("After restoring snapshot: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("After restoring snapshot: mob cells count",
|
||||
regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
|
||||
assertEquals("After restoring snapshot: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After restoring snapshot: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After restoring snapshot: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("After restoring snapshot: mob cells count", regionNum * cellNumPerRow
|
||||
* rowNumPerRegion, countMobCells(table));
|
||||
assertEquals("After restoring snapshot: family1 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After restoring snapshot: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After restoring snapshot: family1 del file count", 0,
|
||||
countFiles(tableName, false, family1));
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After restoring snapshot: family2 del file count", 0,
|
||||
countFiles(tableName, false, family2));
|
||||
assertEquals("After restoring snapshot: family1 hfilelink count", regionNum*count,
|
||||
countHFileLinks(family1));
|
||||
assertEquals("After restoring snapshot: family2 hfilelink count", 0,
|
||||
countHFileLinks(family2));
|
||||
countFiles(tableName, false, family2));
|
||||
assertEquals("After restoring snapshot: family1 hfilelink count", regionNum * count,
|
||||
countHFileLinks(family1));
|
||||
assertEquals("After restoring snapshot: family2 hfilelink count", 0, countHFileLinks(family2));
|
||||
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After second compaction: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("After second compaction: mob cells count",
|
||||
regionNum*cellNumPerRow*rowNumPerRegion, countMobCells(hTable));
|
||||
assertEquals("After second compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("After second compaction: mob cells count", regionNum * cellNumPerRow
|
||||
* rowNumPerRegion, countMobCells(table));
|
||||
assertEquals("After second compaction: family1 mob file count", regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After second compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After second compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After second compaction: family1 del file count", 0,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After second compaction: family2 del file count", 0,
|
||||
|
|
@ -423,115 +299,86 @@ public class TestMobCompactor {
|
|||
assertRefFileNameEqual(family1);
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionFromAdmin() throws Exception {
|
||||
resetConf();
|
||||
int mergeSize = 5000;
|
||||
// change the mob compaction merge size
|
||||
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
|
||||
assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
|
||||
countMobCells(hTable));
|
||||
assertEquals("Before deleting: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before deleting: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
|
||||
createDelFile();
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("Before compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
|
||||
int largeFilesCount = countLargeFiles(5000, family1);
|
||||
// do the mob compaction
|
||||
admin.compact(tableName, hcd1.getName(), Admin.CompactType.MOB);
|
||||
|
||||
waitUntilMobCompactionFinished(tableName);
|
||||
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(hTable));
|
||||
assertEquals("After compaction: family1 mob file count", regionNum + largeFilesCount,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
assertRefFileNameEqual(family1);
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
public void testMajorCompactionFromAdmin() throws Exception {
|
||||
resetConf();
|
||||
int mergeSize = 5000;
|
||||
// change the mob compaction merge size
|
||||
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
|
||||
String tableNameAsString = "testMajorCompactionFromAdmin";
|
||||
SecureRandom rng = new SecureRandom();
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
rng.nextBytes(keyBytes);
|
||||
String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key cfKey = new SecretKeySpec(keyBytes, algorithm);
|
||||
byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
|
||||
conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
|
||||
TableName tableName = TableName.valueOf(tableNameAsString);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
|
||||
hcd1.setMobEnabled(true);
|
||||
hcd1.setMobThreshold(0);
|
||||
hcd1.setEncryptionType(algorithm);
|
||||
hcd1.setEncryptionKey(encryptionKey);
|
||||
HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
|
||||
hcd2.setMobEnabled(true);
|
||||
hcd2.setMobThreshold(0);
|
||||
desc.addFamily(hcd1);
|
||||
desc.addFamily(hcd2);
|
||||
admin.createTable(desc, getSplitKeys());
|
||||
Table table = conn.getTable(tableName);
|
||||
BufferedMutator bufMut = conn.getBufferedMutator(tableName);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
assertEquals("Before deleting: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before deleting: mob cells count", regionNum*cellNumPerRow*rowNumPerRegion,
|
||||
countMobCells(hTable));
|
||||
assertEquals("Before deleting: mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before deleting: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("Before deleting: mob cells count", regionNum * cellNumPerRow * rowNumPerRegion,
|
||||
countMobCells(table));
|
||||
assertEquals("Before deleting: mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
|
||||
createDelFile();
|
||||
createDelFile(table, tableName, Bytes.toBytes(family1), Bytes.toBytes(qf1));
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
assertEquals("Before compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("Before compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
countFiles(tableName, false, family2));
|
||||
|
||||
// do the major mob compaction, it will force all files to compaction
|
||||
admin.majorCompact(tableName, hcd1.getName(), Admin.CompactType.MOB);
|
||||
|
||||
waitUntilMobCompactionFinished(tableName);
|
||||
assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("After compaction: mob rows count", regionNum * (rowNumPerRegion - delRowNum),
|
||||
countMobRows(table));
|
||||
assertEquals("After compaction: mob cells count", regionNum
|
||||
* (cellNumPerRow * rowNumPerRegion - delCellNum), countMobCells(table));
|
||||
assertEquals("After compaction: family1 mob file count", regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After compaction: family1 del file count", 0,
|
||||
countFiles(tableName, false, family1));
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
countFiles(tableName, false, family2));
|
||||
Assert.assertTrue(verifyEncryption(tableName, family1));
|
||||
table.close();
|
||||
}
|
||||
|
||||
@Test(timeout = 300000)
|
||||
public void testScannerOnBulkLoadRefHFiles() throws Exception {
|
||||
resetConf();
|
||||
setUp("testScannerOnBulkLoadRefHFiles");
|
||||
long ts = EnvironmentEdgeManager.currentTime();
|
||||
byte[] key0 = Bytes.toBytes("k0");
|
||||
byte[] key1 = Bytes.toBytes("k1");
|
||||
|
|
@ -548,7 +395,7 @@ public class TestMobCompactor {
|
|||
loadData(admin, bufMut, tableName, new Put[] { put0, put1 });
|
||||
// read the latest cell of key0.
|
||||
Get get = new Get(key0);
|
||||
Result result = hTable.get(get);
|
||||
Result result = table.get(get);
|
||||
Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
|
||||
assertEquals("Before compaction: mob value of k0", newValue0,
|
||||
Bytes.toString(CellUtil.cloneValue(cell)));
|
||||
|
|
@ -556,18 +403,18 @@ public class TestMobCompactor {
|
|||
waitUntilMobCompactionFinished(tableName);
|
||||
// read the latest cell of key0, the cell seqId in bulk loaded file is not reset in the
|
||||
// scanner. The cell that has "new" value is still visible.
|
||||
result = hTable.get(get);
|
||||
result = table.get(get);
|
||||
cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
|
||||
assertEquals("After compaction: mob value of k0", newValue0,
|
||||
Bytes.toString(CellUtil.cloneValue(cell)));
|
||||
// read the ref cell, not read further to the mob cell.
|
||||
get = new Get(key1);
|
||||
get.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(true));
|
||||
result = hTable.get(get);
|
||||
result = table.get(get);
|
||||
cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
|
||||
// the ref name is the new file
|
||||
Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
|
||||
tableName), hcd1.getNameAsString());
|
||||
Path mobFamilyPath = new Path(
|
||||
MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(), tableName), hcd1.getNameAsString());
|
||||
List<Path> paths = new ArrayList<Path>();
|
||||
if (fs.exists(mobFamilyPath)) {
|
||||
FileStatus[] files = fs.listStatus(mobFamilyPath);
|
||||
|
|
@ -585,6 +432,7 @@ public class TestMobCompactor {
|
|||
@Test(timeout = 300000)
|
||||
public void testScannerAfterCompactions() throws Exception {
|
||||
resetConf();
|
||||
setUp("testScannerAfterCompactions");
|
||||
long ts = EnvironmentEdgeManager.currentTime();
|
||||
byte[] key0 = Bytes.toBytes("k0");
|
||||
byte[] key1 = Bytes.toBytes("k1");
|
||||
|
|
@ -605,7 +453,7 @@ public class TestMobCompactor {
|
|||
waitUntilMobCompactionFinished(tableName);
|
||||
// read the latest cell of key1.
|
||||
Get get = new Get(key1);
|
||||
Result result = hTable.get(get);
|
||||
Result result = table.get(get);
|
||||
Cell cell = result.getColumnLatestCell(hcd1.getName(), Bytes.toBytes(qf1));
|
||||
assertEquals("After compaction: mob value", "new", Bytes.toString(CellUtil.cloneValue(cell)));
|
||||
}
|
||||
|
|
@ -748,19 +596,19 @@ public class TestMobCompactor {
|
|||
/**
|
||||
* Gets the number of files.
|
||||
* @param size the size of the file
|
||||
* @param tableName the current table name
|
||||
* @param familyName the family name
|
||||
* @return the number of files large than the size
|
||||
*/
|
||||
private int countLargeFiles(int size, String familyName) throws IOException {
|
||||
Path mobDirPath = MobUtils.getMobFamilyPath(
|
||||
MobUtils.getMobRegionPath(conf, tableName), familyName);
|
||||
private int countLargeFiles(int size, TableName tableName, String familyName) throws IOException {
|
||||
Path mobDirPath = MobUtils.getMobFamilyPath(MobUtils.getMobRegionPath(conf, tableName),
|
||||
familyName);
|
||||
int count = 0;
|
||||
if (fs.exists(mobDirPath)) {
|
||||
FileStatus[] files = fs.listStatus(mobDirPath);
|
||||
for (FileStatus file : files) {
|
||||
// ignore the del files in the mob path
|
||||
if ((!StoreFileInfo.isDelFile(file.getPath()))
|
||||
&& (file.getLen() > size)) {
|
||||
if ((!StoreFileInfo.isDelFile(file.getPath())) && (file.getLen() > size)) {
|
||||
count++;
|
||||
}
|
||||
}
|
||||
|
|
@ -776,9 +624,9 @@ public class TestMobCompactor {
|
|||
if (fileNum <= 0) {
|
||||
throw new IllegalArgumentException();
|
||||
}
|
||||
for (byte k0 : KEYS) {
|
||||
byte[] k = new byte[] { k0 };
|
||||
for (int i = 0; i < fileNum * rowNumPerFile; i++) {
|
||||
for (int i = 0; i < fileNum * rowNumPerFile; i++) {
|
||||
for (byte k0 : KEYS) {
|
||||
byte[] k = new byte[] { k0 };
|
||||
byte[] key = Bytes.add(k, Bytes.toBytes(i));
|
||||
byte[] mobVal = makeDummyData(10 * (i + 1));
|
||||
Put put = new Put(key);
|
||||
|
|
@ -787,10 +635,10 @@ public class TestMobCompactor {
|
|||
put.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf2), mobVal);
|
||||
put.addColumn(Bytes.toBytes(family2), Bytes.toBytes(qf1), mobVal);
|
||||
table.mutate(put);
|
||||
if ((i + 1) % rowNumPerFile == 0) {
|
||||
table.flush();
|
||||
admin.flush(tableName);
|
||||
}
|
||||
}
|
||||
if ((i + 1) % rowNumPerFile == 0) {
|
||||
table.flush();
|
||||
admin.flush(tableName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -805,30 +653,30 @@ public class TestMobCompactor {
|
|||
/**
|
||||
* delete the row, family and cell to create the del file
|
||||
*/
|
||||
private void createDelFile() throws IOException, InterruptedException {
|
||||
private void createDelFile(Table table, TableName tableName, byte[] family, byte[] qf)
|
||||
throws IOException, InterruptedException {
|
||||
for (byte k0 : KEYS) {
|
||||
byte[] k = new byte[] { k0 };
|
||||
// delete a family
|
||||
byte[] key1 = Bytes.add(k, Bytes.toBytes(0));
|
||||
Delete delete1 = new Delete(key1);
|
||||
delete1.addFamily(Bytes.toBytes(family1));
|
||||
hTable.delete(delete1);
|
||||
delete1.addFamily(family);
|
||||
table.delete(delete1);
|
||||
// delete one row
|
||||
byte[] key2 = Bytes.add(k, Bytes.toBytes(2));
|
||||
Delete delete2 = new Delete(key2);
|
||||
hTable.delete(delete2);
|
||||
table.delete(delete2);
|
||||
// delete one cell
|
||||
byte[] key3 = Bytes.add(k, Bytes.toBytes(4));
|
||||
Delete delete3 = new Delete(key3);
|
||||
delete3.addColumn(Bytes.toBytes(family1), Bytes.toBytes(qf1));
|
||||
hTable.delete(delete3);
|
||||
admin.flush(tableName);
|
||||
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(
|
||||
Bytes.toBytes(tableNameAsString));
|
||||
for (HRegion region : regions) {
|
||||
region.waitForFlushesAndCompactions();
|
||||
region.compact(true);
|
||||
}
|
||||
delete3.addColumn(family, qf);
|
||||
table.delete(delete3);
|
||||
}
|
||||
admin.flush(tableName);
|
||||
List<HRegion> regions = TEST_UTIL.getHBaseCluster().getRegions(tableName);
|
||||
for (HRegion region : regions) {
|
||||
region.waitForFlushesAndCompactions();
|
||||
region.compact(true);
|
||||
}
|
||||
}
|
||||
/**
|
||||
|
|
@ -880,7 +728,7 @@ public class TestMobCompactor {
|
|||
scan.addFamily(Bytes.toBytes(familyName));
|
||||
// Do not retrieve the mob data when scanning
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
|
||||
ResultScanner results = hTable.getScanner(scan);
|
||||
ResultScanner results = table.getScanner(scan);
|
||||
Path mobFamilyPath = new Path(MobUtils.getMobRegionPath(TEST_UTIL.getConfiguration(),
|
||||
tableName), familyName);
|
||||
List<Path> actualFilePaths = new ArrayList<>();
|
||||
|
|
|
|||
Loading…
Reference in New Issue