Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "HBASE-14684 Try to remove all MiniMapReduceCluster in unit tests" 

This reverts commit b0965b1805.
This commit is contained in:
chenheng 2015-12-22 16:10:58 +08:00
parent a51dc02ed3
commit cf059e54b5
20 changed files with 55 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
View File

@ -78,7 +78,6 @@ public abstract class MultiTableInputFormatTestBase {
public static void setUpBeforeClass() throws Exception {
// switch TIF to log at DEBUG level
TEST_UTIL.enableDebug(MultiTableInputFormatBase.class);
TEST_UTIL.setJobWithoutMRCluster();
// start mini hbase cluster
TEST_UTIL.startMiniCluster(3);
// create and fill table
@ -93,10 +92,13 @@ public abstract class MultiTableInputFormatTestBase {
}
}
}
// start MR cluster
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
View File

@ -60,7 +60,6 @@ public abstract class TableSnapshotInputFormatTestBase {
public void setupCluster() throws Exception {
setupConf(UTIL.getConfiguration());
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster(NUM_REGION_SERVERS, true);
rootDir = UTIL.getHBaseCluster().getMaster().getMasterFileSystem().getRootDir();
fs = rootDir.getFileSystem(UTIL.getConfiguration());
@ -159,12 +158,14 @@ public abstract class TableSnapshotInputFormatTestBase {
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
setupCluster();
util.startMiniMapReduceCluster();
try {
Path tableDir = util.getDataTestDirOnTestFS(snapshotName);
TableName tableName = TableName.valueOf("testWithMapReduce");
testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
expectedNumSplits, shutdownCluster);
} finally {
util.shutdownMiniMapReduceCluster();
tearDownCluster();
}
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
View File

@ -60,14 +60,15 @@ public class TestCellCounter {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
FileUtil.fullyDelete(new File(OUTPUT_DIR));
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
View File

@ -62,12 +62,13 @@ public class TestCopyTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.setJobWithoutMRCluster();
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

11
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
View File

@ -390,7 +390,6 @@ public class TestHFileOutputFormat {
byte[][] splitKeys = generateRandomSplitKeys(4);
HBaseAdmin admin = null;
try {
util.setJobWithoutMRCluster();
util.startMiniCluster();
Path testDir = util.getDataTestDirOnTestFS("testLocalMRIncrementalLoad");
admin = util.getHBaseAdmin();
@ -404,6 +403,7 @@ public class TestHFileOutputFormat {
assertEquals("Should make 5 regions", numRegions, 5);
// Generate the bulk load files
util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table",
@ -472,6 +472,7 @@ public class TestHFileOutputFormat {
tableDigestBefore, util.checksumRows(table));
} finally {
if (admin != null) admin.close();
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -918,7 +919,6 @@ public class TestHFileOutputFormat {
generateRandomStartKeys(5);
try {
util.setJobWithoutMRCluster();
util.startMiniCluster();
final FileSystem fs = util.getDFSCluster().getFileSystem();
HBaseAdmin admin = new HBaseAdmin(conf);
@ -935,6 +935,7 @@ public class TestHFileOutputFormat {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@ -973,6 +974,7 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -984,10 +986,9 @@ public class TestHFileOutputFormat {
generateRandomStartKeys(5);
try {
util.setJobWithoutMRCluster();
util.startMiniCluster();
Path testDir = util.getDataTestDirOnTestFS("testExcludeMinorCompaction");
final FileSystem fs = util.getTestFileSystem();
final FileSystem fs = util.getDFSCluster().getFileSystem();
HBaseAdmin admin = new HBaseAdmin(conf);
HTable table = util.createTable(TABLE_NAME, FAMILIES);
assertEquals("Should start with empty table", 0, util.countRows(table));
@ -1014,6 +1015,7 @@ public class TestHFileOutputFormat {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table, testDir);
// Perform the actual load
@ -1049,6 +1051,7 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}

11
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
View File

@ -421,7 +421,6 @@ public class TestHFileOutputFormat2 {
for (int i = 0; i < hostCount; ++i) {
hostnames[i] = "datanode_" + i;
}
util.setJobWithoutMRCluster();
util.startMiniCluster(1, hostCount, hostnames);
HTable table = util.createTable(TABLE_NAME, FAMILIES, splitKeys);
@ -432,6 +431,7 @@ public class TestHFileOutputFormat2 {
assertEquals("Should make " + regionNum + " regions", numRegions, regionNum);
// Generate the bulk load files
util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table.getTableDescriptor(), table.getRegionLocator(), testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table", 0, util.countRows(table));
@ -511,6 +511,7 @@ public class TestHFileOutputFormat2 {
} finally {
testDir.getFileSystem(conf).delete(testDir, true);
util.deleteTable(TABLE_NAME);
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -955,7 +956,7 @@ public class TestHFileOutputFormat2 {
Configuration conf = util.getConfiguration();
conf.setInt("hbase.hstore.compaction.min", 2);
generateRandomStartKeys(5);
util.setJobWithoutMRCluster();
util.startMiniCluster();
try (Connection conn = ConnectionFactory.createConnection();
Admin admin = conn.getAdmin()) {
@ -973,6 +974,7 @@ public class TestHFileOutputFormat2 {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@ -1014,6 +1016,7 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -1023,7 +1026,7 @@ public class TestHFileOutputFormat2 {
Configuration conf = util.getConfiguration();
conf.setInt("hbase.hstore.compaction.min", 2);
generateRandomStartKeys(5);
util.setJobWithoutMRCluster();
util.startMiniCluster();
try (Connection conn = ConnectionFactory.createConnection(conf);
Admin admin = conn.getAdmin()){
@ -1055,6 +1058,7 @@ public class TestHFileOutputFormat2 {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir);
@ -1094,6 +1098,7 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
View File

@ -34,12 +34,13 @@ public class TestHRegionPartitioner {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
View File

@ -56,12 +56,13 @@ public class TestHashTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.setJobWithoutMRCluster();
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
View File

@ -107,14 +107,15 @@ public class TestImportExport {
public static void beforeClass() throws Exception {
// Up the handlers; this test needs more than usual.
UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR =
new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
View File

@ -102,12 +102,14 @@ public class TestImportTSVWithOperationAttributes implements Configurable {
conf = util.getConfiguration();
conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
util.setJobWithoutMRCluster();
util.startMiniCluster();
Admin admin = new HBaseAdmin(util.getConfiguration());
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
View File

@ -88,12 +88,13 @@ public class TestImportTSVWithTTLs implements Configurable {
// need this where the default hfile version is not 3 (i.e. 0.98)
conf.setInt("hfile.format.version", 3);
conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
util.setJobWithoutMRCluster();
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
View File

@ -103,12 +103,13 @@ public class TestImportTsv implements Configurable {
@BeforeClass
public static void provisionCluster() throws Exception {
util.setJobWithoutMRCluster();
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultiTableInputFormat.java
View File

@ -71,7 +71,6 @@ public class TestMultiTableInputFormat {
// switch TIF to log at DEBUG level
TEST_UTIL.enableDebug(MultiTableInputFormat.class);
TEST_UTIL.enableDebug(MultiTableInputFormatBase.class);
TEST_UTIL.setJobWithoutMRCluster();
// start mini hbase cluster
TEST_UTIL.startMiniCluster(3);
// create and fill table
@ -82,10 +81,13 @@ public class TestMultiTableInputFormat {
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
}
}
// start MR cluster
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
View File

@ -72,17 +72,18 @@ public class TestMultithreadedTableMapper {
public static void beforeClass() throws Exception {
// Up the handlers; this test needs more than usual.
UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster();
HTable table =
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
View File

@ -80,8 +80,8 @@ public class TestRowCounter {
@BeforeClass
public static void setUpBeforeClass()
throws Exception {
TEST_UTIL.setJobWithoutMRCluster();
TEST_UTIL.startMiniCluster();
TEST_UTIL.startMiniMapReduceCluster();
Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
writeRows(table);
table.close();
@ -94,6 +94,7 @@ public class TestRowCounter {
public static void tearDownAfterClass()
throws Exception {
TEST_UTIL.shutdownMiniCluster();
TEST_UTIL.shutdownMiniMapReduceCluster();
}
/**

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
View File

@ -64,12 +64,13 @@ public class TestSyncTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.setJobWithoutMRCluster();
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
View File

@ -77,18 +77,20 @@ public class TestTableInputFormat {
private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static MiniMRCluster mrCluster;
static final byte[] FAMILY = Bytes.toBytes("family");
private static final byte[][] columns = new byte[][] { FAMILY };
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster();
mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
View File

@ -79,16 +79,17 @@ public abstract class TestTableMapReduceBase {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster();
HTable table =
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
View File

@ -83,7 +83,6 @@ public class TestTimeRangeMapRed {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.setJobWithoutMRCluster();
UTIL.startMiniCluster();
}
@ -168,6 +167,7 @@ public class TestTimeRangeMapRed {
private void runTestOnTable()
throws IOException, InterruptedException, ClassNotFoundException {
UTIL.startMiniMapReduceCluster();
Job job = null;
try {
job = new Job(UTIL.getConfiguration(), "test123");
@ -184,6 +184,7 @@ public class TestTimeRangeMapRed {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
UTIL.shutdownMiniMapReduceCluster();
if (job != null) {
FileUtil.fullyDelete(
new File(job.getConfiguration().get("hadoop.tmp.dir")));

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
View File

@ -70,12 +70,13 @@ public class TestWALPlayer {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.setJobWithoutMRCluster();
cluster = TEST_UTIL.startMiniCluster();
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}