MAPREDUCE-6632. Master.getMasterAddress() should be updated to use YARN-4629 (templedf via rkanter)
This commit is contained in:
parent
40acacee08
commit
4fc632ae19
|
@ -37,7 +37,11 @@
|
|||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-common</artifactId>
|
||||
<artifactId>hadoop-yarn-client</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
<artifactId>hadoop-yarn-common</artifactId>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hadoop</groupId>
|
||||
|
|
|
@ -19,75 +19,45 @@
|
|||
package org.apache.hadoop.mapred;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import org.apache.commons.logging.Log;
|
||||
import org.apache.commons.logging.LogFactory;
|
||||
import org.apache.hadoop.classification.InterfaceAudience.Private;
|
||||
import org.apache.hadoop.classification.InterfaceStability.Unstable;
|
||||
import org.apache.hadoop.conf.Configuration;
|
||||
import org.apache.hadoop.mapreduce.MRConfig;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.security.SecurityUtil;
|
||||
import org.apache.hadoop.yarn.conf.HAUtil;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import org.apache.hadoop.yarn.client.util.YarnClientUtils;
|
||||
|
||||
@Private
|
||||
@Unstable
|
||||
public class Master {
|
||||
|
||||
private static final Log LOG = LogFactory.getLog(Master.class);
|
||||
|
||||
public enum State {
|
||||
INITIALIZING, RUNNING;
|
||||
}
|
||||
|
||||
public static String getMasterUserName(Configuration conf) {
|
||||
String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
|
||||
return conf.get(MRConfig.MASTER_USER_NAME);
|
||||
}
|
||||
else {
|
||||
return conf.get(YarnConfiguration.RM_PRINCIPAL);
|
||||
}
|
||||
}
|
||||
|
||||
public static InetSocketAddress getMasterAddress(Configuration conf) {
|
||||
String masterAddress;
|
||||
String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
|
||||
masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
|
||||
return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
|
||||
} else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
|
||||
HAUtil.isHAEnabled(conf)) {
|
||||
YarnConfiguration yarnConf = new YarnConfiguration(conf);
|
||||
if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
|
||||
String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
|
||||
if (rmIds != null && rmIds.length > 0) {
|
||||
// If RM_HA_ID is not configured, use the first one.
|
||||
// Because any valid RM HA ID should work.
|
||||
yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
|
||||
} else {
|
||||
LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
|
||||
}
|
||||
}
|
||||
return yarnConf.getSocketAddr(
|
||||
YarnConfiguration.RM_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_PORT);
|
||||
} else {
|
||||
return conf.getSocketAddr(
|
||||
YarnConfiguration.RM_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_ADDRESS,
|
||||
YarnConfiguration.DEFAULT_RM_PORT);
|
||||
}
|
||||
public static String getMasterAddress(Configuration conf) {
|
||||
String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
|
||||
|
||||
return NetUtils.createSocketAddr(masterAddress, 8012,
|
||||
MRConfig.MASTER_ADDRESS).getHostName();
|
||||
}
|
||||
|
||||
public static String getMasterPrincipal(Configuration conf)
|
||||
throws IOException {
|
||||
String masterHostname = getMasterAddress(conf).getHostName();
|
||||
// get kerberos principal for use as delegation token renewer
|
||||
return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname);
|
||||
public static String getMasterPrincipal(Configuration conf)
|
||||
throws IOException {
|
||||
String masterPrincipal;
|
||||
String framework = conf.get(MRConfig.FRAMEWORK_NAME,
|
||||
MRConfig.YARN_FRAMEWORK_NAME);
|
||||
|
||||
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
|
||||
String masterAddress = getMasterAddress(conf);
|
||||
// get kerberos principal for use as delegation token renewer
|
||||
masterPrincipal =
|
||||
SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME),
|
||||
masterAddress);
|
||||
} else {
|
||||
masterPrincipal = YarnClientUtils.getRmPrincipal(conf);
|
||||
}
|
||||
|
||||
return masterPrincipal;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
@ -18,13 +18,10 @@
|
|||
|
||||
package org.apache.hadoop.mapred;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
|
||||
import java.net.InetSocketAddress;
|
||||
|
||||
import org.apache.hadoop.mapreduce.MRConfig;
|
||||
import org.apache.hadoop.net.NetUtils;
|
||||
import org.apache.hadoop.yarn.conf.YarnConfiguration;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.fail;
|
||||
import org.junit.Test;
|
||||
|
||||
public class TestMaster {
|
||||
|
@ -33,13 +30,6 @@ public class TestMaster {
|
|||
public void testGetMasterAddress() {
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
|
||||
// Default is yarn framework
|
||||
String masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
|
||||
// no address set so should default to default rm address
|
||||
InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
|
||||
assertEquals(masterHostname, rmAddr.getHostName());
|
||||
|
||||
// Trying invalid master address for classic
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
|
||||
conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
|
||||
|
@ -55,47 +45,7 @@ public class TestMaster {
|
|||
|
||||
// Change master address to a valid value
|
||||
conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042");
|
||||
masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
String masterHostname = Master.getMasterAddress(conf);
|
||||
assertEquals(masterHostname, "bar.com");
|
||||
|
||||
// change framework to yarn
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
|
||||
masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
assertEquals(masterHostname, "foo1.com");
|
||||
|
||||
// change framework to yarn and enable HA
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
|
||||
conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
|
||||
conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
|
||||
conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
|
||||
masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
// If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
|
||||
assertEquals(masterHostname, "rm1.com");
|
||||
conf.set(YarnConfiguration.RM_HA_ID, "rm2");
|
||||
masterHostname = Master.getMasterAddress(conf).getHostName();
|
||||
// If RM_HA_ID is configured, use the given RM_HA_ID.
|
||||
assertEquals(masterHostname, "rm2.com");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetMasterUser() {
|
||||
YarnConfiguration conf = new YarnConfiguration();
|
||||
conf.set(MRConfig.MASTER_USER_NAME, "foo");
|
||||
conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
|
||||
|
||||
// default is yarn framework
|
||||
assertEquals(Master.getMasterUserName(conf), "bar");
|
||||
|
||||
// set framework name to classic
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
|
||||
assertEquals(Master.getMasterUserName(conf), "foo");
|
||||
|
||||
// change framework to yarn
|
||||
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
|
||||
assertEquals(Master.getMasterUserName(conf), "bar");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue