Apache/hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-14684 Try to remove all MiniMapReduceCluster in unit tests; REAPPLY

Browse Source
This commit is contained in:
stack 2015-10-26 08:40:19 -07:00
parent 0c5f4e01f6
commit ff0bc80374
23 changed files with 4 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java
View File

@ -34,6 +34,7 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Connection;
@ -80,7 +81,7 @@ public class TestTableInputFormat {
private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
private static MiniMRCluster mrCluster;
static final byte[] FAMILY = Bytes.toBytes("family");
private static final byte[][] columns = new byte[][] { FAMILY };
@ -88,12 +89,10 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}
@ -344,7 +343,8 @@ public class TestTableInputFormat {
}
void testInputFormat(Class<? extends InputFormat> clazz) throws IOException {
final JobConf job = MapreduceTestingShim.getJobConf(mrCluster);
Configuration conf = UTIL.getConfiguration();
final JobConf job = new JobConf(conf);
job.setInputFormat(clazz);
job.setOutputFormat(NullOutputFormat.class);
job.setMapperClass(ExampleVerifier.class);

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java
View File

@ -89,12 +89,10 @@ public class TestTableMapReduceUtil {
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
presidentsTable = createAndFillTable(TableName.valueOf(TABLE_NAME));
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java
View File

@ -87,13 +87,10 @@ public abstract class MultiTableInputFormatTestBase {
}
}
}
// start MR cluster
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java
View File

@ -155,14 +155,12 @@ public abstract class TableSnapshotInputFormatTestBase {
protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
setupCluster();
util.startMiniMapReduceCluster();
try {
Path tableDir = util.getDataTestDirOnTestFS(snapshotName);
TableName tableName = TableName.valueOf("testWithMapReduce");
testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
expectedNumSplits, shutdownCluster);
} finally {
util.shutdownMiniMapReduceCluster();
tearDownCluster();
}
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java
View File

@ -61,14 +61,12 @@ public class TestCellCounter {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
FileUtil.fullyDelete(new File(OUTPUT_DIR));
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java
View File

@ -63,12 +63,10 @@ public class TestCopyTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

4
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java
View File

@ -405,7 +405,6 @@ public class TestHFileOutputFormat {
assertEquals("HFOF should not touch actual table",
0, util.countRows(table));
// Make sure that a directory was created for every CF
int dir = 0;
for (FileStatus f : testDir.getFileSystem(conf).listStatus(testDir)) {
@ -931,7 +930,6 @@ public class TestHFileOutputFormat {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@ -970,7 +968,6 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -1047,7 +1044,6 @@ public class TestHFileOutputFormat {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}

5
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java
View File

@ -506,7 +506,6 @@ public class TestHFileOutputFormat2 {
} finally {
testDir.getFileSystem(conf).delete(testDir, true);
util.deleteTable(TABLE_NAME);
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -969,7 +968,6 @@ public class TestHFileOutputFormat2 {
// Generate two bulk load files
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
for (int i = 0; i < 2; i++) {
Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@ -1011,7 +1009,6 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}
@ -1053,7 +1050,6 @@ public class TestHFileOutputFormat2 {
// Generate a bulk load file with more rows
conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
true);
util.startMiniMapReduceCluster();
RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir);
@ -1093,7 +1089,6 @@ public class TestHFileOutputFormat2 {
}, 5000);
} finally {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java
View File

@ -35,12 +35,10 @@ public class TestHRegionPartitioner {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java
View File

@ -57,12 +57,10 @@ public class TestHashTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java
View File

@ -108,14 +108,12 @@ public class TestImportExport {
// Up the handlers; this test needs more than usual.
UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
UTIL.startMiniCluster();
UTIL.startMiniMapReduceCluster();
FQ_OUTPUT_DIR =
new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

1
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java
View File

@ -105,7 +105,6 @@ public class TestImportTSVWithOperationAttributes implements Configurable {
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java
View File

@ -89,12 +89,10 @@ public class TestImportTSVWithTTLs implements Configurable {
conf.setInt("hfile.format.version", 3);
conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

1
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java
View File

@ -151,7 +151,6 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java
View File

@ -104,12 +104,10 @@ public class TestImportTsv implements Configurable {
@BeforeClass
public static void provisionCluster() throws Exception {
util.startMiniCluster();
util.startMiniMapReduceCluster();
}
@AfterClass
public static void releaseCluster() throws Exception {
util.shutdownMiniMapReduceCluster();
util.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java
View File

@ -77,13 +77,11 @@ public class TestMultithreadedTableMapper {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java
View File

@ -76,7 +76,6 @@ public class TestRowCounter {
public static void setUpBeforeClass()
throws Exception {
TEST_UTIL.startMiniCluster();
TEST_UTIL.startMiniMapReduceCluster();
Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
writeRows(table);
table.close();
@ -89,7 +88,6 @@ public class TestRowCounter {
public static void tearDownAfterClass()
throws Exception {
TEST_UTIL.shutdownMiniCluster();
TEST_UTIL.shutdownMiniMapReduceCluster();
}
/**

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java
View File

@ -61,12 +61,10 @@ public class TestSyncTable {
@BeforeClass
public static void beforeClass() throws Exception {
TEST_UTIL.startMiniCluster(3);
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java
View File

@ -85,12 +85,10 @@ public class TestTableInputFormat {
@BeforeClass
public static void beforeClass() throws Exception {
UTIL.startMiniCluster();
mrCluster = UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

3
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java
View File

@ -86,13 +86,10 @@ public abstract class TestTableInputFormatScanBase {
// create and fill table
table = TEST_UTIL.createMultiRegionTable(TableName.valueOf(TABLE_NAME), INPUT_FAMILY);
TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
// start MR cluster
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java
View File

@ -80,12 +80,10 @@ public abstract class TestTableMapReduceBase {
UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
OUTPUT_FAMILY });
UTIL.loadTable(table, INPUT_FAMILY, false);
UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
UTIL.shutdownMiniMapReduceCluster();
UTIL.shutdownMiniCluster();
}

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java
View File

@ -167,7 +167,6 @@ public class TestTimeRangeMapRed {
private void runTestOnTable()
throws IOException, InterruptedException, ClassNotFoundException {
UTIL.startMiniMapReduceCluster();
Job job = null;
try {
job = new Job(UTIL.getConfiguration(), "test123");
@ -184,7 +183,6 @@ public class TestTimeRangeMapRed {
// TODO Auto-generated catch block
e.printStackTrace();
} finally {
UTIL.shutdownMiniMapReduceCluster();
if (job != null) {
FileUtil.fullyDelete(
new File(job.getConfiguration().get("hadoop.tmp.dir")));

2
hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java
View File

@ -71,12 +71,10 @@ public class TestWALPlayer {
@BeforeClass
public static void beforeClass() throws Exception {
cluster = TEST_UTIL.startMiniCluster();
TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void afterClass() throws Exception {
TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}