HBASE-14684 Try to remove all MiniMapReduceCluster in unit tests (Heng Chen)

hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableInputFormat.java

@@ -34,6 +34,7 @@ import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.hbase.*;
 import org.apache.hadoop.hbase.Cell;
 import org.apache.hadoop.hbase.client.Connection;
@@ -80,7 +81,7 @@ public class TestTableInputFormat {
   private static final Log LOG = LogFactory.getLog(TestTableInputFormat.class);
 
   private final static HBaseTestingUtility UTIL = new HBaseTestingUtility();
-  private static MiniMRCluster mrCluster;
+
   static final byte[] FAMILY = Bytes.toBytes("family");
 
   private static final byte[][] columns = new byte[][] { FAMILY };
@@ -88,12 +89,10 @@ public class TestTableInputFormat {
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
-    mrCluster = UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 
@@ -344,7 +343,8 @@ public class TestTableInputFormat {
   }
 
   void testInputFormat(Class<? extends InputFormat> clazz) throws IOException {
-    final JobConf job = MapreduceTestingShim.getJobConf(mrCluster);
+    Configuration conf = UTIL.getConfiguration();
+    final JobConf job = new JobConf(conf);
     job.setInputFormat(clazz);
     job.setOutputFormat(NullOutputFormat.class);
     job.setMapperClass(ExampleVerifier.class);

hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableMapReduceUtil.java

@@ -90,12 +90,10 @@ public class TestTableMapReduceUtil {
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
     presidentsTable = createAndFillTable(TableName.valueOf(TABLE_NAME));
-    UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/MultiTableInputFormatTestBase.java

@@ -83,13 +83,10 @@ public abstract class MultiTableInputFormatTestBase {
         TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
       }
     }
-    // start MR cluster
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TableSnapshotInputFormatTestBase.java

@@ -154,14 +154,12 @@ public abstract class TableSnapshotInputFormatTestBase {
   protected void testWithMapReduce(HBaseTestingUtility util, String snapshotName,
       int numRegions, int expectedNumSplits, boolean shutdownCluster) throws Exception {
     setupCluster();
-    util.startMiniMapReduceCluster();
     try {
       Path tableDir = util.getDataTestDirOnTestFS(snapshotName);
       TableName tableName = TableName.valueOf("testWithMapReduce");
       testWithMapReduceImpl(util, tableName, snapshotName, tableDir, numRegions,
         expectedNumSplits, shutdownCluster);
     } finally {
-      util.shutdownMiniMapReduceCluster();
       tearDownCluster();
     }
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCellCounter.java

@@ -63,14 +63,12 @@ public class TestCellCounter {
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
-    UTIL.startMiniMapReduceCluster();
     FQ_OUTPUT_DIR = new Path(OUTPUT_DIR).makeQualified(new LocalFileSystem());
     FileUtil.fullyDelete(new File(OUTPUT_DIR));
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestCopyTable.java

@@ -62,12 +62,10 @@ public class TestCopyTable {
   @BeforeClass
   public static void beforeClass() throws Exception {
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java

@@ -403,7 +403,6 @@ public class TestHFileOutputFormat  {
         assertEquals("Should make 5 regions", numRegions, 5);
 
         // Generate the bulk load files
-        util.startMiniMapReduceCluster();
         runIncrementalPELoad(conf, table, r, testDir);
         // This doesn't write into the table, just makes files
         assertEquals("HFOF should not touch actual table",
@@ -473,7 +472,6 @@ public class TestHFileOutputFormat  {
       assertEquals("Data should remain after reopening of regions",
           tableDigestBefore, util.checksumRows(table));
     } finally {
-      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -937,7 +935,6 @@ public class TestHFileOutputFormat  {
       // Generate two bulk load files
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
-      util.startMiniMapReduceCluster();
 
       try (RegionLocator regionLocator = util.getConnection().getRegionLocator(TABLE_NAME)) {
         for (int i = 0; i < 2; i++) {
@@ -978,7 +975,6 @@ public class TestHFileOutputFormat  {
       }, 5000);
 
     } finally {
-      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -1019,7 +1015,6 @@ public class TestHFileOutputFormat  {
       // Generate a bulk load file with more rows
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
-      util.startMiniMapReduceCluster();
 
       try (RegionLocator locator = util.getConnection().getRegionLocator(TABLE_NAME)) {
         runIncrementalPELoad(conf, table, locator, testDir);
@@ -1058,7 +1053,6 @@ public class TestHFileOutputFormat  {
       }, 5000);
 
     } finally {
-      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java

@@ -430,7 +430,6 @@ public class TestHFileOutputFormat2  {
       assertEquals("Should make " + regionNum + " regions", numRegions, regionNum);
 
       // Generate the bulk load files
-      util.startMiniMapReduceCluster();
       runIncrementalPELoad(conf, table.getTableDescriptor(), r, testDir);
       // This doesn't write into the table, just makes files
       assertEquals("HFOF should not touch actual table", 0, util.countRows(table));
@@ -511,7 +510,6 @@ public class TestHFileOutputFormat2  {
     } finally {
       testDir.getFileSystem(conf).delete(testDir, true);
       util.deleteTable(TABLE_NAME);
-      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -976,7 +974,6 @@ public class TestHFileOutputFormat2  {
       // Generate two bulk load files
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
-      util.startMiniMapReduceCluster();
 
       for (int i = 0; i < 2; i++) {
         Path testDir = util.getDataTestDirOnTestFS("testExcludeAllFromMinorCompaction_" + i);
@@ -1018,7 +1015,6 @@ public class TestHFileOutputFormat2  {
       }, 5000);
 
     } finally {
-      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }
@@ -1060,7 +1056,6 @@ public class TestHFileOutputFormat2  {
       // Generate a bulk load file with more rows
       conf.setBoolean("hbase.mapreduce.hfileoutputformat.compaction.exclude",
           true);
-      util.startMiniMapReduceCluster();
 
       RegionLocator regionLocator = conn.getRegionLocator(TABLE_NAME);
       runIncrementalPELoad(conf, table.getTableDescriptor(), regionLocator, testDir);
@@ -1100,7 +1095,6 @@ public class TestHFileOutputFormat2  {
       }, 5000);
 
     } finally {
-      util.shutdownMiniMapReduceCluster();
       util.shutdownMiniCluster();
     }
   }

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHRegionPartitioner.java

@@ -36,12 +36,10 @@ public class TestHRegionPartitioner {
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
-    UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHashTable.java

@@ -57,12 +57,10 @@ public class TestHashTable {
   @BeforeClass
   public static void beforeClass() throws Exception {
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
   }
   
   @AfterClass
   public static void afterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
   

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportExport.java

@@ -111,14 +111,12 @@ public class TestImportExport {
     // Up the handlers; this test needs more than usual.
     UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
     UTIL.startMiniCluster();
-    UTIL.startMiniMapReduceCluster();
     FQ_OUTPUT_DIR =
       new Path(OUTPUT_DIR).makeQualified(FileSystem.get(UTIL.getConfiguration())).toString();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithOperationAttributes.java

@@ -99,12 +99,10 @@ public class TestImportTSVWithOperationAttributes implements Configurable {
     conf.set("hbase.coprocessor.master.classes", OperationAttributesTestController.class.getName());
     conf.set("hbase.coprocessor.region.classes", OperationAttributesTestController.class.getName());
     util.startMiniCluster();
-    util.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void releaseCluster() throws Exception {
-    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithTTLs.java

@@ -90,12 +90,10 @@ public class TestImportTSVWithTTLs implements Configurable {
     conf.setInt("hfile.format.version", 3);
     conf.set("hbase.coprocessor.region.classes", TTLCheckingObserver.class.getName());
     util.startMiniCluster();
-    util.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void releaseCluster() throws Exception {
-    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTSVWithVisibilityLabels.java

@@ -124,7 +124,6 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
     // Wait for the labels table to become available
     util.waitTableEnabled(VisibilityConstants.LABELS_TABLE_NAME.getName(), 50000);
     createLabels();
-    util.startMiniMapReduceCluster();
   }
 
   private static void createLabels() throws IOException, InterruptedException {
@@ -148,7 +147,6 @@ public class TestImportTSVWithVisibilityLabels implements Configurable {
 
   @AfterClass
   public static void releaseCluster() throws Exception {
-    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestImportTsv.java

@@ -104,12 +104,10 @@ public class TestImportTsv implements Configurable {
   @BeforeClass
   public static void provisionCluster() throws Exception {
     util.startMiniCluster();
-    util.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void releaseCluster() throws Exception {
-    util.shutdownMiniMapReduceCluster();
     util.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestMultithreadedTableMapper.java

@@ -76,13 +76,11 @@ public class TestMultithreadedTableMapper {
         UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
             OUTPUT_FAMILY });
     UTIL.loadTable(table, INPUT_FAMILY, false);
-    UTIL.startMiniMapReduceCluster();
     UTIL.waitUntilAllRegionsAssigned(MULTI_REGION_TABLE_NAME);
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestRowCounter.java

@@ -64,7 +64,6 @@ public class TestRowCounter {
   @BeforeClass
   public static void setUpBeforeClass() throws Exception {
     TEST_UTIL.startMiniCluster();
-    TEST_UTIL.startMiniMapReduceCluster();
     Table table = TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME), Bytes.toBytes(COL_FAM));
     writeRows(table);
     table.close();
@@ -76,7 +75,6 @@ public class TestRowCounter {
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
     TEST_UTIL.shutdownMiniCluster();
-    TEST_UTIL.shutdownMiniMapReduceCluster();
   }
 
   /**

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestSyncTable.java

@@ -60,12 +60,10 @@ public class TestSyncTable {
   @BeforeClass
   public static void beforeClass() throws Exception {
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
   }
   
   @AfterClass
   public static void afterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
   

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormat.java

@@ -84,12 +84,10 @@ public class TestTableInputFormat {
   @BeforeClass
   public static void beforeClass() throws Exception {
     UTIL.startMiniCluster();
-    mrCluster = UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableInputFormatScanBase.java

@@ -84,13 +84,10 @@ public abstract class TestTableInputFormatScanBase {
     // create and fill table
     table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, INPUT_FAMILY);
     TEST_UTIL.loadTable(table, INPUT_FAMILY, false);
-    // start MR cluster
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTableMapReduceBase.java

@@ -79,12 +79,10 @@ public abstract class TestTableMapReduceBase {
         UTIL.createMultiRegionTable(MULTI_REGION_TABLE_NAME, new byte[][] { INPUT_FAMILY,
             OUTPUT_FAMILY });
     UTIL.loadTable(table, INPUT_FAMILY, false);
-    UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    UTIL.shutdownMiniMapReduceCluster();
     UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestTimeRangeMapRed.java

@@ -168,7 +168,6 @@ public class TestTimeRangeMapRed {
 
   private void runTestOnTable()
   throws IOException, InterruptedException, ClassNotFoundException {
-    UTIL.startMiniMapReduceCluster();
     Job job = null;
     try {
       job = new Job(UTIL.getConfiguration(), "test123");
@@ -185,7 +184,6 @@ public class TestTimeRangeMapRed {
       // TODO Auto-generated catch block
       e.printStackTrace();
     } finally {
-      UTIL.shutdownMiniMapReduceCluster();
       if (job != null) {
         FileUtil.fullyDelete(
           new File(job.getConfiguration().get("hadoop.tmp.dir")));

hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestWALPlayer.java

@@ -73,12 +73,10 @@ public class TestWALPlayer {
   @BeforeClass
   public static void beforeClass() throws Exception {
     cluster = TEST_UTIL.startMiniCluster();
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void afterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java

@@ -87,12 +87,10 @@ public class TestExportSnapshot {
   public static void setUpBeforeClass() throws Exception {
     setUpBaseConf(TEST_UTIL.getConfiguration());
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @AfterClass
   public static void tearDownAfterClass() throws Exception {
-    TEST_UTIL.shutdownMiniMapReduceCluster();
     TEST_UTIL.shutdownMiniCluster();
   }
 

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java

@@ -45,7 +45,6 @@ public class TestMobExportSnapshot extends TestExportSnapshot {
   public static void setUpBeforeClass() throws Exception {
     setUpBaseConf(TEST_UTIL.getConfiguration());
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
   }
 
   @Override

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java

@@ -45,7 +45,6 @@ public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
     SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
 
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
 
     // Wait for the ACL table to become available
     TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);

hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java

@@ -46,7 +46,6 @@ public class TestSecureExportSnapshot extends TestExportSnapshot {
     SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
 
     TEST_UTIL.startMiniCluster(3);
-    TEST_UTIL.startMiniMapReduceCluster();
 
     // Wait for the ACL table to become available
     TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);