MAPREDUCE-6632. Master.getMasterAddress() should be updated to use YARN-4629 (templedf via rkanter)

(cherry picked from commit 4fc632ae19)

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml

@@ -35,6 +35,10 @@
   </properties>
 
   <dependencies>
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-yarn-client</artifactId>
+    </dependency>
     <dependency>
       <groupId>org.apache.hadoop</groupId>
       <artifactId>hadoop-yarn-common</artifactId>

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java

@@ -19,75 +19,45 @@
 package org.apache.hadoop.mapred;
 
 import java.io.IOException;
-import java.net.InetSocketAddress;
 
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability.Unstable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.mapreduce.MRConfig;
 import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
 
 @Private
 @Unstable
 public class Master {
-
-  private static final Log LOG = LogFactory.getLog(Master.class);
-
   public enum State {
     INITIALIZING, RUNNING;
   }
 
-  public static String getMasterUserName(Configuration conf) {
+  public static String getMasterAddress(Configuration conf) {
-    String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+    String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
-    if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {    
-      return conf.get(MRConfig.MASTER_USER_NAME);
-    } 
-    else {
-      return conf.get(YarnConfiguration.RM_PRINCIPAL);
-    }
-  }
 
-  public static InetSocketAddress getMasterAddress(Configuration conf) {
+    return NetUtils.createSocketAddr(masterAddress, 8012,
-    String masterAddress;
+            MRConfig.MASTER_ADDRESS).getHostName();
-    String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
-    if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
-      masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
-      return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
-    } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
-        HAUtil.isHAEnabled(conf)) {
-      YarnConfiguration yarnConf = new YarnConfiguration(conf);
-      if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
-        String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
-        if (rmIds != null && rmIds.length > 0) {
-          // If RM_HA_ID is not configured, use the first one.
-          // Because any valid RM HA ID should work.
-          yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
-        } else {
-          LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
-        }
-      }
-      return yarnConf.getSocketAddr(
-          YarnConfiguration.RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_PORT);
-    } else {
-      return conf.getSocketAddr(
-          YarnConfiguration.RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_ADDRESS,
-          YarnConfiguration.DEFAULT_RM_PORT);
-    }
   }
 
   public static String getMasterPrincipal(Configuration conf)
-  throws IOException {
+      throws IOException {
-    String masterHostname = getMasterAddress(conf).getHostName();
+    String masterPrincipal;
-    // get kerberos principal for use as delegation token renewer
+    String framework = conf.get(MRConfig.FRAMEWORK_NAME,
-    return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname);
+            MRConfig.YARN_FRAMEWORK_NAME);
-  }
 
+    if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
+      String masterAddress = getMasterAddress(conf);
+      // get kerberos principal for use as delegation token renewer
+      masterPrincipal =
+          SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME),
+          masterAddress);
+    } else {
+      masterPrincipal = YarnClientUtils.getRmPrincipal(conf);
+    }
+
+    return masterPrincipal;
+  }
 }

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java

@@ -18,13 +18,10 @@
 
 package org.apache.hadoop.mapred;
 
-import static org.junit.Assert.*;
-
-import java.net.InetSocketAddress;
-
 import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.net.NetUtils;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
 import org.junit.Test;
 
 public class TestMaster {
@@ -33,13 +30,6 @@ public class TestMaster {
   public void testGetMasterAddress() {
     YarnConfiguration conf = new YarnConfiguration();
 
-    // Default is yarn framework
-    String masterHostname = Master.getMasterAddress(conf).getHostName();
-    
-    // no address set so should default to default rm address
-    InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
-    assertEquals(masterHostname, rmAddr.getHostName());
-    
     // Trying invalid master address for classic 
     conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
     conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
@@ -55,47 +45,7 @@ public class TestMaster {
 
     // Change master address to a valid value
     conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042");    
-    masterHostname = Master.getMasterAddress(conf).getHostName();
+    String masterHostname = Master.getMasterAddress(conf);
     assertEquals(masterHostname, "bar.com");
-
-    // change framework to yarn
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
-    conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
-    masterHostname = Master.getMasterAddress(conf).getHostName();
-    assertEquals(masterHostname, "foo1.com");
-
-    // change framework to yarn and enable HA
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
-    conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
-    conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
-    conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
-    conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
-    masterHostname = Master.getMasterAddress(conf).getHostName();
-    // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
-    assertEquals(masterHostname, "rm1.com");
-    conf.set(YarnConfiguration.RM_HA_ID, "rm2");
-    masterHostname = Master.getMasterAddress(conf).getHostName();
-    // If RM_HA_ID is configured, use the given RM_HA_ID.
-    assertEquals(masterHostname, "rm2.com");
   }
-
-  @Test 
-  public void testGetMasterUser() {
-    YarnConfiguration conf = new YarnConfiguration();
-    conf.set(MRConfig.MASTER_USER_NAME, "foo");
-    conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
-
-    // default is yarn framework  
-    assertEquals(Master.getMasterUserName(conf), "bar");
-
-    // set framework name to classic
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
-    assertEquals(Master.getMasterUserName(conf), "foo");
-
-    // change framework to yarn
-    conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
-    assertEquals(Master.getMasterUserName(conf), "bar");
-
-  }
-
 }