From b839d04d8e2cc3dcadeddf08203591466650ac3e Mon Sep 17 00:00:00 2001 From: Loknath Priyatham Teja Singamsetty Date: Thu, 1 Sep 2016 21:16:26 +0530 Subject: [PATCH] HBASE-16375 Mapreduce mini cluster using HBaseTestingUtility not setting correct resourcemanager and jobhistory webapp address of MapReduceTestingShim Signed-off-by: Andrew Purtell Amending-Author: Andrew Purtell --- .../hadoop/hbase/HBaseTestingUtility.java | 10 ++++ .../hadoop/hbase/TestHBaseTestingUtility.java | 46 +++++++++++++++++-- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java index 04afb014e35..c164091a422 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java @@ -2529,6 +2529,16 @@ public class HBaseTestingUtility extends HBaseCommonTestingUtility { if (schedulerAddress != null) { conf.set("yarn.resourcemanager.scheduler.address", schedulerAddress); } + String mrJobHistoryWebappAddress = + jobConf.get("mapreduce.jobhistory.webapp.address"); + if (mrJobHistoryWebappAddress != null) { + conf.set("mapreduce.jobhistory.webapp.address", mrJobHistoryWebappAddress); + } + String yarnRMWebappAddress = + jobConf.get("yarn.resourcemanager.webapp.address"); + if (yarnRMWebappAddress != null) { + conf.set("yarn.resourcemanager.webapp.address", yarnRMWebappAddress); + } } /** diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java index 15d4bacc4d9..a00aa85e1f5 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestHBaseTestingUtility.java @@ -24,6 +24,14 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotEquals; import static org.junit.Assert.assertTrue; +import java.io.File; +import java.io.IOException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Random; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -34,22 +42,18 @@ import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Result; import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; import org.apache.hadoop.hbase.testclassification.LargeTests; import org.apache.hadoop.hbase.testclassification.MiscTests; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.zookeeper.MiniZooKeeperCluster; import org.apache.hadoop.hdfs.MiniDFSCluster; -import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil; import org.junit.Test; import org.junit.experimental.categories.Category; import org.mockito.Mockito; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; -import java.io.File; -import java.util.List; -import java.util.Random; - /** * Test our testing utility class */ @@ -443,4 +447,36 @@ public class TestHBaseTestingUtility { assertEquals(nonDefaultRegionServerPort , htu.getConfiguration().getInt(HConstants.REGIONSERVER_PORT, 0)); } + + @Test public void testMRYarnConfigsPopulation() throws IOException { + Map dummyProps = new HashMap<>(); + dummyProps.put("mapreduce.jobtracker.address", "dummyhost:11234"); + dummyProps.put("yarn.resourcemanager.address", "dummyhost:11235"); + dummyProps.put("mapreduce.jobhistory.address", "dummyhost:11236"); + dummyProps.put("yarn.resourcemanager.scheduler.address", "dummyhost:11237"); + dummyProps.put("mapreduce.jobhistory.webapp.address", "dummyhost:11238"); + dummyProps.put("yarn.resourcemanager.webapp.address", "dummyhost:11239"); + + HBaseTestingUtility hbt = new HBaseTestingUtility(); + + // populate the mr props to the Configuration instance + for (Entry entry : dummyProps.entrySet()) { + hbt.getConfiguration().set(entry.getKey(), entry.getValue()); + } + + for (Entry entry : dummyProps.entrySet()) { + assertTrue("The Configuration for key " + entry.getKey() +" and value: " + entry.getValue() + + " is not populated correctly", hbt.getConfiguration().get(entry.getKey()).equals(entry.getValue())); + } + + hbt.startMiniMapReduceCluster(); + + // Confirm that MiniMapReduceCluster overwrites the mr properties and updates the Configuration + for (Entry entry : dummyProps.entrySet()) { + assertFalse("The MR prop: " + entry.getValue() + " is not overwritten when map reduce mini"+ + "cluster is started", hbt.getConfiguration().get(entry.getKey()).equals(entry.getValue())); + } + + hbt.shutdownMiniMapReduceCluster(); + } }