HBASE-14567 Disable hanging test TestMobCompactor and TestMobSweeper
This commit is contained in:
stack
parent
7c11a06b25
commit
2e8e32f746
|
|
@ -169,82 +169,13 @@ public class TestMobCompactor {
|
|||
fs.delete(TEST_UTIL.getDataTestDir(), true);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionWithoutDelFilesWithNamespace() throws Exception {
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionWithoutDelFiles() throws Exception {
|
||||
resetConf();
|
||||
// create a table with namespace
|
||||
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
|
||||
String tableNameAsString = "ns:testCompactionWithoutDelFilesWithNamespace";
|
||||
String tableNameAsString = "ns:testCompactionWithoutDelFiles";
|
||||
admin.createNamespace(namespaceDescriptor);
|
||||
TableName tableName = TableName.valueOf(tableNameAsString);
|
||||
HColumnDescriptor hcd1 = new HColumnDescriptor(family1);
|
||||
hcd1.setMobEnabled(true);
|
||||
hcd1.setMobThreshold(0);
|
||||
hcd1.setMaxVersions(4);
|
||||
HColumnDescriptor hcd2 = new HColumnDescriptor(family2);
|
||||
hcd2.setMobEnabled(true);
|
||||
hcd2.setMobThreshold(0);
|
||||
hcd2.setMaxVersions(4);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
desc.addFamily(hcd1);
|
||||
desc.addFamily(hcd2);
|
||||
admin.createTable(desc, getSplitKeys());
|
||||
BufferedMutator bufMut= conn.getBufferedMutator(tableName);
|
||||
Table table = conn.getTable(tableName);
|
||||
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("Before compaction: mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
|
||||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("After compaction: mob file count", regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
|
||||
table.close();
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
admin.deleteNamespace("ns");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionWithoutDelFiles() throws Exception {
|
||||
resetConf();
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
|
||||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob file count", regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionWithoutDelFilesAndWithEncryption() throws Exception {
|
||||
resetConf();
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
SecureRandom rng = new SecureRandom();
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
|
|
@ -253,7 +184,6 @@ public class TestMobCompactor {
|
|||
Key cfKey = new SecretKeySpec(keyBytes, algorithm);
|
||||
byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
|
||||
conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
|
||||
String tableNameAsString = "testCompactionWithoutDelFilesAndWithEncryption";
|
||||
TableName tableName = TableName.valueOf(tableNameAsString);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(family1);
|
||||
|
|
@ -269,15 +199,15 @@ public class TestMobCompactor {
|
|||
desc.addFamily(hcd);
|
||||
desc.addFamily(hcd2);
|
||||
admin.createTable(desc, getSplitKeys());
|
||||
Table hTable = conn.getTable(tableName);
|
||||
Table table = conn.getTable(tableName);
|
||||
BufferedMutator bufMut = conn.getBufferedMutator(tableName);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
int rowNumPerRegion = count * rowNumPerFile;
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("Before compaction: mob file count", regionNum * count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
|
|
@ -285,17 +215,23 @@ public class TestMobCompactor {
|
|||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob rows count", regionNum * rowNumPerRegion,
|
||||
countMobRows(table));
|
||||
assertEquals("After compaction: mob file count", regionNum,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: del file count", 0, countFiles(tableName, false, family1));
|
||||
Assert.assertTrue(verifyEncryption(tableName, family1));
|
||||
table.close();
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
admin.deleteNamespace("ns");
|
||||
}
|
||||
|
||||
@Test
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionWithDelFiles() throws Exception {
|
||||
resetConf();
|
||||
int batchSize = 2;
|
||||
conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
|
|
@ -333,7 +269,7 @@ public class TestMobCompactor {
|
|||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("After compaction: family1 mob file count", regionNum,
|
||||
assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize),
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
|
|
@ -344,8 +280,8 @@ public class TestMobCompactor {
|
|||
assertRefFileNameEqual(family1);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionWithDelFilesAndNotMergeAllFiles() throws Exception {
|
||||
@Test(timeout = 300000)
|
||||
public void testMinorCompactionWithDelFiles() throws Exception {
|
||||
resetConf();
|
||||
int mergeSize = 5000;
|
||||
// change the mob compaction merge size
|
||||
|
|
@ -399,57 +335,7 @@ public class TestMobCompactor {
|
|||
countFiles(tableName, false, family2));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionWithDelFilesAndWithSmallCompactionBatchSize() throws Exception {
|
||||
resetConf();
|
||||
int batchSize = 2;
|
||||
conf.setInt(MobConstants.MOB_COMPACTION_BATCH_SIZE, batchSize);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
int rowNumPerRegion = count*rowNumPerFile;
|
||||
|
||||
assertEquals("Before deleting: mob row count", regionNum*rowNumPerRegion,
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before deleting: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before deleting: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
|
||||
createDelFile();
|
||||
|
||||
assertEquals("Before compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("Before compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("Before compaction: family1 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("Before compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("Before compaction: family1 del file count", regionNum,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("Before compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
|
||||
// do the mob compaction
|
||||
MobCompactor compactor = new PartitionedMobCompactor(conf, fs, tableName, hcd1, pool);
|
||||
compactor.compact();
|
||||
|
||||
assertEquals("After compaction: mob rows count", regionNum*(rowNumPerRegion-delRowNum),
|
||||
countMobRows(hTable));
|
||||
assertEquals("After compaction: mob cells count",
|
||||
regionNum*(cellNumPerRow*rowNumPerRegion-delCellNum), countMobCells(hTable));
|
||||
assertEquals("After compaction: family1 mob file count", regionNum*(count/batchSize),
|
||||
countFiles(tableName, true, family1));
|
||||
assertEquals("After compaction: family2 mob file count", regionNum*count,
|
||||
countFiles(tableName, true, family2));
|
||||
assertEquals("After compaction: family1 del file count", 0,
|
||||
countFiles(tableName, false, family1));
|
||||
assertEquals("After compaction: family2 del file count", regionNum,
|
||||
countFiles(tableName, false, family2));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionWithHFileLink() throws IOException, InterruptedException {
|
||||
resetConf();
|
||||
int count = 4;
|
||||
|
|
@ -537,8 +423,12 @@ public class TestMobCompactor {
|
|||
assertRefFileNameEqual(family1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Test(timeout = 300000)
|
||||
public void testCompactionFromAdmin() throws Exception {
|
||||
resetConf();
|
||||
int mergeSize = 5000;
|
||||
// change the mob compaction merge size
|
||||
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
|
|
@ -588,8 +478,12 @@ public class TestMobCompactor {
|
|||
assertRefFileNameEqual(family1);
|
||||
}
|
||||
|
||||
@Test
|
||||
@Test(timeout = 300000)
|
||||
public void testMajorCompactionFromAdmin() throws Exception {
|
||||
resetConf();
|
||||
int mergeSize = 5000;
|
||||
// change the mob compaction merge size
|
||||
conf.setLong(MobConstants.MOB_COMPACTION_MERGEABLE_THRESHOLD, mergeSize);
|
||||
int count = 4;
|
||||
// generate mob files
|
||||
loadData(admin, bufMut, tableName, count, rowNumPerFile);
|
||||
|
|
@ -635,8 +529,9 @@ public class TestMobCompactor {
|
|||
countFiles(tableName, false, family2));
|
||||
}
|
||||
|
||||
@Test
|
||||
@Test(timeout = 300000)
|
||||
public void testScannerOnBulkLoadRefHFiles() throws Exception {
|
||||
resetConf();
|
||||
long ts = EnvironmentEdgeManager.currentTime();
|
||||
byte[] key0 = Bytes.toBytes("k0");
|
||||
byte[] key1 = Bytes.toBytes("k1");
|
||||
|
|
@ -687,8 +582,9 @@ public class TestMobCompactor {
|
|||
.getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Test(timeout = 300000)
|
||||
public void testScannerAfterCompactions() throws Exception {
|
||||
resetConf();
|
||||
long ts = EnvironmentEdgeManager.currentTime();
|
||||
byte[] key0 = Bytes.toBytes("k0");
|
||||
byte[] key1 = Bytes.toBytes("k1");
|
||||
|
|
|
|||
|
|
@ -1,368 +0,0 @@
|
|||
/**
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.hadoop.hbase.mob.mapreduce;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.security.Key;
|
||||
import java.security.SecureRandom;
|
||||
import java.util.Random;
|
||||
import java.util.Set;
|
||||
import java.util.TreeSet;
|
||||
|
||||
import javax.crypto.spec.SecretKeySpec;
|
||||
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.fs.FileStatus;
|
||||
import org.apache.hadoop.fs.Path;
|
||||
import org.apache.hadoop.hbase.HBaseTestingUtility;
|
||||
import org.apache.hadoop.hbase.HColumnDescriptor;
|
||||
import org.apache.hadoop.hbase.HConstants;
|
||||
import org.apache.hadoop.hbase.HTableDescriptor;
|
||||
import org.apache.hadoop.hbase.NamespaceDescriptor;
|
||||
import org.apache.hadoop.hbase.TableName;
|
||||
import org.apache.hadoop.hbase.client.*;
|
||||
import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting;
|
||||
import org.apache.hadoop.hbase.io.crypto.aes.AES;
|
||||
import org.apache.hadoop.hbase.io.hfile.CacheConfig;
|
||||
import org.apache.hadoop.hbase.io.hfile.HFile;
|
||||
import org.apache.hadoop.hbase.mob.MobConstants;
|
||||
import org.apache.hadoop.hbase.mob.MobUtils;
|
||||
import org.apache.hadoop.hbase.regionserver.BloomType;
|
||||
import org.apache.hadoop.hbase.regionserver.StoreFile;
|
||||
import org.apache.hadoop.hbase.security.EncryptionUtil;
|
||||
import org.apache.hadoop.hbase.security.User;
|
||||
import org.apache.hadoop.hbase.testclassification.MediumTests;
|
||||
import org.apache.hadoop.hbase.util.Bytes;
|
||||
import org.apache.hadoop.util.ToolRunner;
|
||||
import org.junit.After;
|
||||
import org.junit.AfterClass;
|
||||
import org.junit.Assert;
|
||||
import org.junit.Before;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.Test;
|
||||
import org.junit.experimental.categories.Category;
|
||||
|
||||
@Category(MediumTests.class)
|
||||
public class TestMobSweeper {
|
||||
private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
|
||||
private TableName tableName;
|
||||
private final static String row = "row_";
|
||||
private final static String family = "family";
|
||||
private final static String column = "column";
|
||||
private static Table table;
|
||||
private static BufferedMutator bufMut;
|
||||
private static Admin admin;
|
||||
|
||||
private Random random = new Random();
|
||||
@BeforeClass
|
||||
public static void setUpBeforeClass() throws Exception {
|
||||
TEST_UTIL.getConfiguration().setInt("hbase.master.info.port", 0);
|
||||
TEST_UTIL.getConfiguration().setBoolean("hbase.regionserver.info.port.auto", true);
|
||||
// avoid major compactions
|
||||
TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.min", 15);
|
||||
// avoid major compactions
|
||||
TEST_UTIL.getConfiguration().setInt("hbase.hstore.compaction.max", 30);
|
||||
TEST_UTIL.getConfiguration().setInt("hfile.format.version", 3);
|
||||
TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_KEYPROVIDER_CONF_KEY,
|
||||
KeyProviderForTesting.class.getName());
|
||||
TEST_UTIL.getConfiguration().set(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, "hbase");
|
||||
|
||||
TEST_UTIL.startMiniCluster();
|
||||
|
||||
TEST_UTIL.startMiniMapReduceCluster();
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
public static void tearDownAfterClass() throws Exception {
|
||||
TEST_UTIL.shutdownMiniCluster();
|
||||
TEST_UTIL.shutdownMiniMapReduceCluster();
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setUp() throws Exception {
|
||||
long tid = System.currentTimeMillis();
|
||||
tableName = TableName.valueOf("testSweeper" + tid);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||
hcd.setMobEnabled(true);
|
||||
hcd.setMobThreshold(0);
|
||||
hcd.setMaxVersions(4);
|
||||
desc.addFamily(hcd);
|
||||
|
||||
admin = TEST_UTIL.getHBaseAdmin();
|
||||
admin.createTable(desc);
|
||||
Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
|
||||
table = c.getTable(tableName);
|
||||
bufMut = c.getBufferedMutator(tableName);
|
||||
}
|
||||
|
||||
@After
|
||||
public void tearDown() throws Exception {
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
admin.close();
|
||||
}
|
||||
|
||||
private Path getMobFamilyPath(Configuration conf, TableName tableName,
|
||||
String familyName) {
|
||||
Path p = new Path(MobUtils.getMobRegionPath(conf, tableName),
|
||||
familyName);
|
||||
return p;
|
||||
}
|
||||
|
||||
private String mergeString(Set<String> set) {
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (String s : set)
|
||||
sb.append(s);
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private void generateMobTable(Admin admin, BufferedMutator table, TableName tableName, int count,
|
||||
int flushStep) throws IOException, InterruptedException {
|
||||
if (count <= 0 || flushStep <= 0)
|
||||
return;
|
||||
int index = 0;
|
||||
for (int i = 0; i < count; i++) {
|
||||
byte[] mobVal = new byte[101*1024];
|
||||
random.nextBytes(mobVal);
|
||||
|
||||
Put put = new Put(Bytes.toBytes(row + i));
|
||||
put.addColumn(Bytes.toBytes(family), Bytes.toBytes(column), mobVal);
|
||||
table.mutate(put);
|
||||
if (index++ % flushStep == 0) {
|
||||
table.flush();
|
||||
admin.flush(tableName);
|
||||
}
|
||||
}
|
||||
table.flush();
|
||||
admin.flush(tableName);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSweeper() throws Exception {
|
||||
int count = 10;
|
||||
//create table and generate 10 mob files
|
||||
generateMobTable(admin, bufMut, tableName, count, 1);
|
||||
//get mob files
|
||||
Path mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
|
||||
FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
|
||||
// mobFileSet0 stores the original mob files
|
||||
TreeSet<String> mobFilesSet = new TreeSet<String>();
|
||||
for (FileStatus status : fileStatuses) {
|
||||
mobFilesSet.add(status.getPath().getName());
|
||||
}
|
||||
|
||||
//scan the table, retreive the references
|
||||
Scan scan = new Scan();
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE));
|
||||
ResultScanner rs = table.getScanner(scan);
|
||||
TreeSet<String> mobFilesScanned = new TreeSet<String>();
|
||||
for (Result res : rs) {
|
||||
byte[] valueBytes = res.getValue(Bytes.toBytes(family),
|
||||
Bytes.toBytes(column));
|
||||
mobFilesScanned.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT,
|
||||
valueBytes.length - Bytes.SIZEOF_INT));
|
||||
}
|
||||
//there should be 10 mob files
|
||||
assertEquals(10, mobFilesScanned.size());
|
||||
//check if we store the correct reference of mob files
|
||||
assertEquals(mergeString(mobFilesSet), mergeString(mobFilesScanned));
|
||||
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
conf.setLong(SweepJob.MOB_SWEEP_JOB_DELAY, 24 * 60 * 60 * 1000);
|
||||
|
||||
String[] args = new String[2];
|
||||
args[0] = tableName.getNameAsString();
|
||||
args[1] = family;
|
||||
assertEquals(0, ToolRunner.run(conf, new Sweeper(), args));
|
||||
|
||||
mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
|
||||
fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
|
||||
mobFilesSet = new TreeSet<String>();
|
||||
for (FileStatus status : fileStatuses) {
|
||||
mobFilesSet.add(status.getPath().getName());
|
||||
}
|
||||
assertEquals(10, mobFilesSet.size());
|
||||
|
||||
scan = new Scan();
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE));
|
||||
rs = table.getScanner(scan);
|
||||
TreeSet<String> mobFilesScannedAfterJob = new TreeSet<String>();
|
||||
for (Result res : rs) {
|
||||
byte[] valueBytes = res.getValue(Bytes.toBytes(family), Bytes.toBytes(
|
||||
column));
|
||||
mobFilesScannedAfterJob.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT,
|
||||
valueBytes.length - Bytes.SIZEOF_INT));
|
||||
}
|
||||
assertEquals(10, mobFilesScannedAfterJob.size());
|
||||
|
||||
fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
|
||||
mobFilesSet = new TreeSet<String>();
|
||||
for (FileStatus status : fileStatuses) {
|
||||
mobFilesSet.add(status.getPath().getName());
|
||||
}
|
||||
assertEquals(10, mobFilesSet.size());
|
||||
assertEquals(true, mobFilesScannedAfterJob.iterator().next()
|
||||
.equalsIgnoreCase(mobFilesSet.iterator().next()));
|
||||
}
|
||||
|
||||
private void testCompactionDelaySweeperInternal(Table table, BufferedMutator bufMut,
|
||||
TableName tableName, boolean encrypted) throws Exception {
|
||||
int count = 10;
|
||||
//create table and generate 10 mob files
|
||||
generateMobTable(admin, bufMut, tableName, count, 1);
|
||||
//get mob files
|
||||
Path mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
|
||||
FileStatus[] fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
|
||||
// mobFileSet0 stores the orignal mob files
|
||||
TreeSet<String> mobFilesSet = new TreeSet<String>();
|
||||
for (FileStatus status : fileStatuses) {
|
||||
mobFilesSet.add(status.getPath().getName());
|
||||
}
|
||||
|
||||
//scan the table, retreive the references
|
||||
Scan scan = new Scan();
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE));
|
||||
ResultScanner rs = table.getScanner(scan);
|
||||
TreeSet<String> mobFilesScanned = new TreeSet<String>();
|
||||
for (Result res : rs) {
|
||||
byte[] valueBytes = res.getValue(Bytes.toBytes(family),
|
||||
Bytes.toBytes(column));
|
||||
mobFilesScanned.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT,
|
||||
valueBytes.length - Bytes.SIZEOF_INT));
|
||||
}
|
||||
//there should be 10 mob files
|
||||
assertEquals(10, mobFilesScanned.size());
|
||||
//check if we store the correct reference of mob files
|
||||
assertEquals(mergeString(mobFilesSet), mergeString(mobFilesScanned));
|
||||
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
conf.setLong(SweepJob.MOB_SWEEP_JOB_DELAY, 0);
|
||||
String[] args = new String[2];
|
||||
args[0] = tableName.getNameAsString();
|
||||
args[1] = family;
|
||||
assertEquals(0, ToolRunner.run(conf, new Sweeper(), args));
|
||||
|
||||
mobFamilyPath = getMobFamilyPath(TEST_UTIL.getConfiguration(), tableName, family);
|
||||
fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
|
||||
mobFilesSet = new TreeSet<String>();
|
||||
for (FileStatus status : fileStatuses) {
|
||||
mobFilesSet.add(status.getPath().getName());
|
||||
}
|
||||
assertEquals(1, mobFilesSet.size());
|
||||
|
||||
scan = new Scan();
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_RAW, Bytes.toBytes(Boolean.TRUE));
|
||||
scan.setAttribute(MobConstants.MOB_SCAN_REF_ONLY, Bytes.toBytes(Boolean.TRUE));
|
||||
rs = table.getScanner(scan);
|
||||
TreeSet<String> mobFilesScannedAfterJob = new TreeSet<String>();
|
||||
for (Result res : rs) {
|
||||
byte[] valueBytes = res.getValue(Bytes.toBytes(family), Bytes.toBytes(
|
||||
column));
|
||||
mobFilesScannedAfterJob.add(Bytes.toString(valueBytes, Bytes.SIZEOF_INT,
|
||||
valueBytes.length - Bytes.SIZEOF_INT));
|
||||
}
|
||||
assertEquals(1, mobFilesScannedAfterJob.size());
|
||||
|
||||
fileStatuses = TEST_UTIL.getTestFileSystem().listStatus(mobFamilyPath);
|
||||
Path lastFilePath = null;
|
||||
mobFilesSet = new TreeSet<String>();
|
||||
for (FileStatus status : fileStatuses) {
|
||||
mobFilesSet.add(status.getPath().getName());
|
||||
lastFilePath = status.getPath();
|
||||
}
|
||||
assertEquals(1, mobFilesSet.size());
|
||||
assertEquals(true, mobFilesScannedAfterJob.iterator().next()
|
||||
.equalsIgnoreCase(mobFilesSet.iterator().next()));
|
||||
if (encrypted) {
|
||||
// assert the encryption context
|
||||
CacheConfig cacheConf = new CacheConfig(conf);
|
||||
StoreFile sf = new StoreFile(TEST_UTIL.getTestFileSystem(), lastFilePath, conf, cacheConf,
|
||||
BloomType.NONE);
|
||||
HFile.Reader reader = sf.createReader().getHFileReader();
|
||||
byte[] encryptionKey = reader.getTrailer().getEncryptionKey();
|
||||
Assert.assertTrue(null != encryptionKey);
|
||||
Assert.assertTrue(reader.getFileContext().getEncryptionContext().getCipher().getName()
|
||||
.equals(HConstants.CIPHER_AES));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionDelaySweeperWithEncryption() throws Exception {
|
||||
Configuration conf = TEST_UTIL.getConfiguration();
|
||||
SecureRandom rng = new SecureRandom();
|
||||
byte[] keyBytes = new byte[AES.KEY_LENGTH];
|
||||
rng.nextBytes(keyBytes);
|
||||
String algorithm = conf.get(HConstants.CRYPTO_KEY_ALGORITHM_CONF_KEY, HConstants.CIPHER_AES);
|
||||
Key cfKey = new SecretKeySpec(keyBytes, algorithm);
|
||||
byte[] encryptionKey = EncryptionUtil.wrapKey(conf,
|
||||
conf.get(HConstants.CRYPTO_MASTERKEY_NAME_CONF_KEY, User.getCurrent().getShortName()), cfKey);
|
||||
String tableNameAsString = "testCompactionDelaySweeperWithEncryption";
|
||||
TableName tableName = TableName.valueOf(tableNameAsString);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||
hcd.setMobEnabled(true);
|
||||
hcd.setMobThreshold(0);
|
||||
hcd.setMaxVersions(4);
|
||||
hcd.setEncryptionType(algorithm);
|
||||
hcd.setEncryptionKey(encryptionKey);
|
||||
desc.addFamily(hcd);
|
||||
admin.createTable(desc);
|
||||
Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
|
||||
BufferedMutator bufMut = c.getBufferedMutator(tableName);
|
||||
Table table = c.getTable(tableName);
|
||||
testCompactionDelaySweeperInternal(table, bufMut, tableName, true);
|
||||
table.close();
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionDelaySweeper() throws Exception {
|
||||
testCompactionDelaySweeperInternal(table, bufMut, tableName, false);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testCompactionDelaySweeperWithNamespace() throws Exception {
|
||||
// create a table with namespace
|
||||
NamespaceDescriptor namespaceDescriptor = NamespaceDescriptor.create("ns").build();
|
||||
admin.createNamespace(namespaceDescriptor);
|
||||
String tableNameAsString = "ns:testSweeperWithNamespace";
|
||||
TableName tableName = TableName.valueOf(tableNameAsString);
|
||||
HTableDescriptor desc = new HTableDescriptor(tableName);
|
||||
HColumnDescriptor hcd = new HColumnDescriptor(family);
|
||||
hcd.setMobEnabled(true);
|
||||
hcd.setMobThreshold(0);
|
||||
hcd.setMaxVersions(4);
|
||||
desc.addFamily(hcd);
|
||||
admin.createTable(desc);
|
||||
Connection c = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
|
||||
BufferedMutator bufMut = c.getBufferedMutator(tableName);
|
||||
Table table = c.getTable(tableName);
|
||||
testCompactionDelaySweeperInternal(table, bufMut, tableName, false);
|
||||
table.close();
|
||||
admin.disableTable(tableName);
|
||||
admin.deleteTable(tableName);
|
||||
admin.deleteNamespace("ns");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue