Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Revert "HBASE-14684 Try to remove all MiniMapReduceCluster in unit tests (Heng Chen)" 

This reverts commit ef7f8467af.
This commit is contained in:
stack 2015-10-24 10:00:27 -07:00
parent 418c4422f7
commit f9efeaad1d
27 changed files with 63 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
View File

@ -34,7 +34,6 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Connection;
@ -81,7 +80,7 @@ public class TestTableInputFormat {
private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static MiniMRCluster mrCluster;
static final byte[] FAMILY = Bytes.toBytes("family");
private static final byte[][] columns = new byte[][] { FAMILY };
@ -89,10 +88,12 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
@ -343,8 +344,7 @@ public class TestTableInputFormat {
}
void testInputFormat(Class<? extends InputFormat> clazz) throws IOException {
Configuration conf = UTIL.getConfiguration();
final JobConf job = new JobConf(conf);
final JobConf job = MapreduceTestingShim.getJobConf(mrCluster);
job.setInputFormat(clazz);
job.setOutputFormat(NullOutputFormat.class);
job.setMapperClass(ExampleVerifier.class);

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
View File

@ -90,10 +90,12 @@ public class TestTableMapReduceUtil {
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
presidentsTable = createAndFillTable(TableName.valueOf(TABLE_NAME));
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
View File

@ -83,10 +83,13 @@ public abstract class MultiTableInputFormatTestBase {
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
}
}
// start MR cluster
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
View File

@ -154,12 +154,14 @@ public abstract class TableSnapshotInputFormatTestBase {
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
setupCluster();
util.startMiniMapReduceCluster();
try {
Path tableDir = util.getDataTestDirOnTestFS(snapshotName);
TableName tableName = TableName.valueOf("testWithMapReduce");
testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
expectedNumSplits, shutdownCluster);
} finally {
util.shutdownMiniMapReduceCluster();
tearDownCluster();
}
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
View File

@ -63,12 +63,14 @@ public class TestCellCounter {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
FileUtil.fullyDelete(new File(OUTPUT_DIR));
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
View File

@ -62,10 +62,12 @@ public class TestCopyTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

6
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
View File

@ -403,6 +403,7 @@ public class TestHFileOutputFormat {
assertEquals("Should make 5 regions", numRegions, 5);
// Generate the bulk load files
util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table, r, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table",
@ -472,6 +473,7 @@ public class TestHFileOutputFormat {
assertEquals("Data should remain after reopening of regions",
tableDigestBefore, util.checksumRows(table));
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -935,6 +937,7 @@ public class TestHFileOutputFormat {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
try (RegionLocator regionLocator = util.getConnection().getRegionLocator(TABLE_NAME)) {
for (int i = 0; i < 2; i++) {
@ -975,6 +978,7 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -1015,6 +1019,7 @@ public class TestHFileOutputFormat {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
try (RegionLocator locator = util.getConnection().getRegionLocator(TABLE_NAME)) {
runIncrementalPELoad(conf, table, locator, testDir);
@ -1053,6 +1058,7 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}

6
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
View File

@ -430,6 +430,7 @@ public class TestHFileOutputFormat2 {
assertEquals("Should make " + regionNum + " regions", numRegions, regionNum);
// Generate the bulk load files
util.startMiniMapReduceCluster();
runIncrementalPELoad(conf, table.getTableDescriptor(), r, testDir);
// This doesn't write into the table, just makes files
assertEquals("HFOF should not touch actual table", 0, util.countRows(table));
@ -510,6 +511,7 @@ public class TestHFileOutputFormat2 {
} finally {
testDir.getFileSystem(conf).delete(testDir, true);
util.deleteTable(TABLE_NAME);
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -974,6 +976,7 @@ public class TestHFileOutputFormat2 {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@ -1015,6 +1018,7 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -1056,6 +1060,7 @@ public class TestHFileOutputFormat2 {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir);
@ -1095,6 +1100,7 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
View File

@ -36,10 +36,12 @@ public class TestHRegionPartitioner {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
View File

@ -57,10 +57,12 @@ public class TestHashTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
View File

@ -111,12 +111,14 @@ public class TestImportExport {
// Up the handlers; this test needs more than usual.
UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR =
new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
View File

@ -99,10 +99,12 @@ public class TestImportTSVWithOperationAttributes implements Configurable {
conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
View File

@ -90,10 +90,12 @@ public class TestImportTSVWithTTLs implements Configurable {
conf.setInt("hfile.format.version", 3);
conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
View File

@ -124,6 +124,7 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
// Wait for the labels table to become available
util.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
createLabels();
util.startMiniMapReduceCluster();
}
private static void createLabels() throws IOException, InterruptedException {
@ -147,6 +148,7 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
View File

@ -104,10 +104,12 @@ public class TestImportTsv implements Configurable {
@BeforeClass
public static void provisionCluster() throws Exception {
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
View File

@ -76,11 +76,13 @@ public class TestMultithreadedTableMapper {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
View File

@ -64,6 +64,7 @@ public class TestRowCounter {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
TEST_UTIL.startMiniCluster();
TEST_UTIL.startMiniMapReduceCluster();
Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
writeRows(table);
table.close();
@ -75,6 +76,7 @@ public class TestRowCounter {
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
TEST_UTIL.shutdownMiniMapReduceCluster();
}
/**

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
View File

@ -60,10 +60,12 @@ public class TestSyncTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
View File

@ -84,10 +84,12 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
View File

@ -84,10 +84,13 @@ public abstract class TestTableInputFormatScanBase {
// create and fill table
table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, INPUT_FAMILY);
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
// start MR cluster
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
View File

@ -79,10 +79,12 @@ public abstract class TestTableMapReduceBase {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
View File

@ -168,6 +168,7 @@ public class TestTimeRangeMapRed {
private void runTestOnTable()
throws IOException, InterruptedException, ClassNotFoundException {
UTIL.startMiniMapReduceCluster();
Job job = null;
try {
job = new Job(UTIL.getConfiguration(), "test123");
@ -184,6 +185,7 @@ public class TestTimeRangeMapRed {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
UTIL.shutdownMiniMapReduceCluster();
if (job != null) {
FileUtil.fullyDelete(
new File(job.getConfiguration().get("hadoop.tmp.dir")));

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
View File

@ -73,10 +73,12 @@ public class TestWALPlayer {
@BeforeClass
public static void beforeClass() throws Exception {
cluster = TEST_UTIL.startMiniCluster();
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
View File

@ -87,10 +87,12 @@ public class TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

1
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
View File

@ -45,6 +45,7 @@ public class TestMobExportSnapshot extends TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@Override

1
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
View File

@ -45,6 +45,7 @@ public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);

1
hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
View File

@ -46,6 +46,7 @@ public class TestSecureExportSnapshot extends TestExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);