Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-16634 Speedup TestExportSnapshot

This commit is contained in:
Matteo Bertozzi 2016-09-15 18:29:09 -07:00
parent 216e847366
commit e19632a19a
7 changed files with 54 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
View File

@ -38,6 +38,20 @@ public class MobSnapshotTestingUtils {
public static void createMobTable(final HBaseTestingUtility util,
final TableName tableName, int regionReplication,
final byte[]... families) throws IOException, InterruptedException {
createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(),
regionReplication, families);
}
public static void createPreSplitMobTable(final HBaseTestingUtility util,
final TableName tableName, int nRegions, final byte[]... families)
throws IOException, InterruptedException {
createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions),
1, families);
}
private static void createMobTable(final HBaseTestingUtility util,
final TableName tableName, final byte[][] splitKeys, int regionReplication,
final byte[]... families) throws IOException, InterruptedException {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setRegionReplication(regionReplication);
for (byte[] family : families) {
@ -46,7 +60,6 @@ public class MobSnapshotTestingUtils {
hcd.setMobThreshold(0L);
htd.addFamily(hcd);
}
byte[][] splitKeys = SnapshotTestingUtils.getSplitKeys();
util.getHBaseAdmin().createTable(htd, splitKeys);
SnapshotTestingUtils.waitForTableToBeOnline(util, tableName);
assertEquals((splitKeys.length + 1) * regionReplication, util

34
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
View File

@ -82,9 +82,10 @@ import com.google.protobuf.ServiceException;
*/
@InterfaceAudience.Private
public final class SnapshotTestingUtils {
private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
private static byte[] KEYS = Bytes.toBytes("0123456789");
// default number of regions (and keys) given by getSplitKeys() and createTable()
private static byte[] KEYS = Bytes.toBytes("0123456");
private SnapshotTestingUtils() {
// private constructor for utility class
@ -750,23 +751,32 @@ public final class SnapshotTestingUtils {
}
public static void createTable(final HBaseTestingUtility util, final TableName tableName,
int regionReplication, final byte[]... families) throws IOException, InterruptedException {
int regionReplication, int nRegions, final byte[]... families)
throws IOException, InterruptedException {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setRegionReplication(regionReplication);
for (byte[] family : families) {
HColumnDescriptor hcd = new HColumnDescriptor(family);
htd.addFamily(hcd);
}
byte[][] splitKeys = getSplitKeys();
byte[][] splitKeys = getSplitKeys(nRegions);
util.createTable(htd, splitKeys);
assertEquals((splitKeys.length + 1) * regionReplication,
util.getHBaseAdmin().getTableRegions(tableName).size());
}
public static byte[][] getSplitKeys() {
byte[][] splitKeys = new byte[KEYS.length-2][];
return getSplitKeys(KEYS.length);
}
public static byte[][] getSplitKeys(int nRegions) {
nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
final byte[][] splitKeys = new byte[nRegions-1][];
final int step = KEYS.length / nRegions;
int keyIndex = 1;
for (int i = 0; i < splitKeys.length; ++i) {
splitKeys[i] = new byte[] { KEYS[i+1] };
splitKeys[i] = new byte[] { KEYS[keyIndex] };
keyIndex += step;
}
return splitKeys;
}
@ -776,6 +786,16 @@ public final class SnapshotTestingUtils {
createTable(util, tableName, 1, families);
}
public static void createTable(final HBaseTestingUtility util, final TableName tableName,
final int regionReplication, final byte[]... families) throws IOException, InterruptedException {
createTable(util, tableName, regionReplication, KEYS.length, families);
}
public static void createPreSplitTable(final HBaseTestingUtility util, final TableName tableName,
final int nRegions, final byte[]... families) throws IOException, InterruptedException {
createTable(util, tableName, 1, nRegions, families);
}
public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
byte[]... families) throws IOException, InterruptedException {
BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
@ -861,6 +881,6 @@ public final class SnapshotTestingUtils {
}
}
}
assert(set.size() == getSplitKeys().length + 1);
assertEquals(getSplitKeys().length + 1, set.size());
}
}

8
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@ -62,7 +62,7 @@ import org.junit.rules.TestRule;
/**
* Test Export Snapshot Tool
*/
@Category({VerySlowRegionServerTests.class, MediumTests.class})
@Category({VerySlowRegionServerTests.class, LargeTests.class})
public class TestExportSnapshot {
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().
withTimeout(this.getClass()).withLookingForStuckThread(true).build();
@ -91,12 +91,10 @@ public class TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
@ -127,7 +125,7 @@ public class TestExportSnapshot {
}
protected void createTable() throws Exception {
SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
}
@After

8
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
View File

@ -325,8 +325,8 @@ public class TestFlushSnapshotFromClient {
int numRegionsAfterMerge = numRegions - 2;
admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(),
regions.get(2).getEncodedNameAsBytes(), true);
admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(),
regions.get(6).getEncodedNameAsBytes(), true);
admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(),
regions.get(5).getEncodedNameAsBytes(), true);
// Verify that there's one region less
waitRegionsAfterMerge(numRegionsAfterMerge);
@ -366,8 +366,8 @@ public class TestFlushSnapshotFromClient {
int numRegionsAfterMerge = numRegions - 2;
admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(),
regions.get(2).getEncodedNameAsBytes(), true);
admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(),
regions.get(6).getEncodedNameAsBytes(), true);
admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(),
regions.get(5).getEncodedNameAsBytes(), true);
waitRegionsAfterMerge(numRegionsAfterMerge);
assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());

8
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
View File

@ -25,14 +25,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
/**
* Test Export Snapshot Tool
*/
@Category(MediumTests.class)
@Category({VerySlowRegionServerTests.class, LargeTests.class})
public class TestMobExportSnapshot extends TestExportSnapshot {
private final Log LOG = LogFactory.getLog(getClass());
@ -45,12 +46,11 @@ public class TestMobExportSnapshot extends TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@Override
protected void createTable() throws Exception {
MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, 1, FAMILY);
MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY);
}
@Override

4
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
View File

@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.access.AccessControlLists;
@ -31,7 +32,7 @@ import org.junit.experimental.categories.Category;
/**
* Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode.
*/
@Category(LargeTests.class)
@Category({VerySlowRegionServerTests.class, LargeTests.class})
public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@ -45,7 +46,6 @@ public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);

1
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
View File

@ -51,7 +51,6 @@ public class TestSecureExportSnapshot extends TestExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);