Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in secure setup with kerberos. Contributed by Omkar Vinit Joshi. 

svn merge --ignore-ancestry -c 1517101 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1517102 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2013-08-24 02:48:17 +00:00
parent 35f60e71de
commit a42a5e153c
6 changed files with 89 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryClientService.java
View File

@ -80,6 +80,7 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@ -148,8 +149,14 @@ public class HistoryClientService extends AbstractService {
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_ADDRESS,
JHAdminConfig.DEFAULT_MR_HISTORY_WEBAPP_PORT);
// NOTE: there should be a .at(InetSocketAddress)
WebApps.$for("jobhistory", HistoryClientService.class, this, "ws")
.with(conf).at(NetUtils.getHostPortString(bindAddress)).start(webApp);
WebApps
.$for("jobhistory", HistoryClientService.class, this, "ws")
.with(conf)
.withHttpSpnegoKeytabKey(
YarnConfiguration.JHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
.withHttpSpnegoPrincipalKey(
YarnConfiguration.JHS_WEBAPP_SPNEGO_USER_NAME_KEY)
.at(NetUtils.getHostPortString(bindAddress)).start(webApp);
conf.updateConnectAddr(JHAdminConfig.MR_HISTORY_WEBAPP_ADDRESS,
webApp.getListenerAddress());
}

3
hadoop-yarn-project/CHANGES.txt
View File

@ -72,6 +72,9 @@ Release 2.1.1-beta - UNRELEASED
YARN-1082. Create base directories on HDFS after RM login to ensure RM
recovery doesn't fail in secure mode. (vinodkv via acmurthy)
YARN-1085. Modified YARN and MR2 web-apps to do HTTP authentication in
secure setup with kerberos. (Omkar Vinit Joshi via vinodkv)
Release 2.1.0-beta - 2013-08-22
INCOMPATIBLE CHANGES

20
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
View File

@ -205,6 +205,12 @@ public class YarnConfiguration extends Configuration {
public static final String RM_KEYTAB =
RM_PREFIX + "keytab";
public static final String RM_WEBAPP_SPNEGO_USER_NAME_KEY =
RM_PREFIX + "webapp.spnego-principal";
public static final String RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY =
RM_PREFIX + "webapp.spengo-keytab-file";
/** How long to wait until a container is considered dead.*/
public static final String RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS =
RM_PREFIX + "rm.container-allocation.expiry-interval-ms";
@ -599,7 +605,13 @@ public class YarnConfiguration extends Configuration {
public static final String NM_USER_HOME_DIR =
NM_PREFIX + "user-home-dir";
public static final String NM_WEBAPP_SPNEGO_USER_NAME_KEY =
NM_PREFIX + "webapp.spnego-principal";
public static final String NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
NM_PREFIX + "webapp.spnego-keytab-file";
public static final String DEFAULT_NM_USER_HOME_DIR= "/home/";
////////////////////////////////
@ -729,6 +741,12 @@ public class YarnConfiguration extends Configuration {
// Other Configs
////////////////////////////////
public static final String JHS_WEBAPP_SPNEGO_USER_NAME_KEY =
"jobhistoryserver.webapp.spnego-principal";
public static final String JHS_WEBAPP_SPNEGO_KEYTAB_FILE_KEY =
"jobhistoryserver.webapp.spnego-keytab-file";
/**
* The interval of the yarn client's querying application state after
* application submission. The unit is millisecond.

41
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
View File

@ -33,6 +33,8 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@ -65,7 +67,6 @@ import com.google.inject.servlet.GuiceFilter;
@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
public class WebApps {
static final Logger LOG = LoggerFactory.getLogger(WebApps.class);
public static class Builder<T> {
static class ServletStruct {
public Class<? extends HttpServlet> clazz;
@ -82,6 +83,8 @@ public class WebApps {
boolean findPort = false;
Configuration conf;
boolean devMode = false;
private String spnegoPrincipalKey;
private String spnegoKeytabKey;
private final HashSet<ServletStruct> servlets = new HashSet<ServletStruct>();
private final HashMap<String, Object> attributes = new HashMap<String, Object>();
@ -135,6 +138,16 @@ public class WebApps {
this.conf = conf;
return this;
}
public Builder<T> withHttpSpnegoPrincipalKey(String spnegoPrincipalKey) {
this.spnegoPrincipalKey = spnegoPrincipalKey;
return this;
}
public Builder<T> withHttpSpnegoKeytabKey(String spnegoKeytabKey) {
this.spnegoKeytabKey = spnegoKeytabKey;
return this;
}
public Builder<T> inDevMode() {
devMode = true;
@ -197,8 +210,30 @@ public class WebApps {
}
}
HttpServer server =
new HttpServer(name, bindAddress, port, findPort, conf,
new AdminACLsManager(conf).getAdminAcl(), null, webapp.getServePathSpecs());
new HttpServer(name, bindAddress, port, findPort, conf,
new AdminACLsManager(conf).getAdminAcl(), null,
webapp.getServePathSpecs()) {
{
if (UserGroupInformation.isSecurityEnabled()) {
boolean initSpnego = true;
if (spnegoPrincipalKey == null || spnegoPrincipalKey.isEmpty()) {
LOG.warn("Principal for spnego filter is not set");
initSpnego = false;
}
if (spnegoKeytabKey == null || spnegoKeytabKey.isEmpty()) {
LOG.warn("Keytab for spnego filter is not set");
initSpnego = false;
}
if (initSpnego) {
LOG.info("Initializing spnego filter with principal key : "
+ spnegoPrincipalKey + " keytab key : "
+ spnegoKeytabKey);
initSpnego(conf, spnegoPrincipalKey, spnegoKeytabKey);
}
}
}
};
for(ServletStruct struct: servlets) {
server.addServlet(struct.name, struct.spec, struct.clazz);
}

11
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
View File

@ -59,8 +59,15 @@ public class WebServer extends AbstractService {
LOG.info("Instantiating NMWebApp at " + bindAddress);
try {
this.webApp =
WebApps.$for("node", Context.class, this.nmContext, "ws")
.at(bindAddress).with(getConfig()).start(this.nmWebApp);
WebApps
.$for("node", Context.class, this.nmContext, "ws")
.at(bindAddress)
.with(getConfig())
.withHttpSpnegoPrincipalKey(
YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY)
.start(this.nmWebApp);
this.port = this.webApp.httpServer().getPort();
} catch (Exception e) {
String msg = "NMWebapps failed to start.";

14
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
View File

@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
@ -573,9 +574,16 @@ public class ResourceManager extends CompositeService implements Recoverable {
protected void startWepApp() {
Builder<ApplicationMasterService> builder =
WebApps.$for("cluster", ApplicationMasterService.class, masterService, "ws").at(
this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS));
WebApps
.$for("cluster", ApplicationMasterService.class, masterService,
"ws")
.with(conf)
.withHttpSpnegoPrincipalKey(
YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY)
.withHttpSpnegoKeytabKey(
YarnConfiguration.RM_WEBAPP_SPENGO_KEYTAB_FILE_KEY)
.at(this.conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS,
YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS));
String proxyHostAndPort = YarnConfiguration.getProxyHostAndPort(conf);
if(YarnConfiguration.getRMWebAppHostAndPort(conf).
equals(proxyHostAndPort)) {