Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-2233. Implemented ResourceManager web-services to create, renew and cancel delegation tokens. Contributed by Varun Vasudev. 

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1610876 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Vinod Kumar Vavilapalli 2014-07-15 23:00:17 +00:00
parent 2d91a0c0f1
commit 030580387a
8 changed files with 1410 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
hadoop-common-project/hadoop-auth/pom.xml
View File

@ -139,6 +139,17 @@
<attach>true</attach>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>

3
hadoop-yarn-project/CHANGES.txt
View File

@ -95,6 +95,9 @@ Release 2.5.0 - UNRELEASED
YARN-1713. Added get-new-app and submit-app functionality to RM web services.
(Varun Vasudev via vinodkv)
YARN-2233. Implemented ResourceManager web-services to create, renew and
cancel delegation tokens. (Varun Vasudev via vinodkv)
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via

15
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
View File

@ -192,6 +192,21 @@
<scope>test</scope>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<scope>test</scope>
<type>test-jar</type>
<version>${project.version}</version>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>com.sun.jersey.jersey-test-framework</groupId>

12
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java
View File

@ -29,8 +29,10 @@
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@ -193,4 +195,14 @@ public void recover(RMState rmState) throws Exception {
addPersistedDelegationToken(entry.getKey(), entry.getValue());
}
}
public long getRenewDate(RMDelegationTokenIdentifier ident)
throws InvalidToken {
DelegationTokenInformation info = currentTokens.get(ident);
if (info == null) {
throw new InvalidToken("token (" + ident.toString()
+ ") can't be found in cache");
}
return info.getRenewDate();
}
}

279
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
View File

@ -22,6 +22,7 @@
import java.lang.reflect.UndeclaredThrowableException;
import java.security.AccessControlException;
import java.nio.ByteBuffer;
import java.security.Principal;
import java.security.PrivilegedExceptionAction;
import java.util.Arrays;
import java.util.Collection;
@ -36,6 +37,7 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.GET;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
@ -57,6 +59,8 @@
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authorize.AuthorizationException;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@ -67,6 +71,13 @@
import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
@ -85,6 +96,7 @@
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
@ -109,6 +121,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
@ -118,6 +131,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@ -139,6 +153,9 @@ public class RMWebServices {
private final Configuration conf;
private @Context HttpServletResponse response;
public final static String DELEGATION_TOKEN_HEADER =
"Hadoop-YARN-RM-Delegation-Token";
@Inject
public RMWebServices(final ResourceManager rm, Configuration conf) {
this.rm = rm;
@ -147,11 +164,7 @@ public RMWebServices(final ResourceManager rm, Configuration conf) {
protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
// Check for the authorization.
String remoteUser = hsr.getRemoteUser();
UserGroupInformation callerUGI = null;
if (remoteUser != null) {
callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
}
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI != null
&& !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
ApplicationAccessType.VIEW_APP, app.getUser(),
@ -626,7 +639,7 @@ public AppAttemptsInfo getAppAttempts(@PathParam("appid") String appId) {
public AppState getAppState(@Context HttpServletRequest hsr,
@PathParam("appid") String appId) throws AuthorizationException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
String userName = "";
if (callerUGI != null) {
userName = callerUGI.getUserName();
@ -661,7 +674,7 @@ public Response updateAppState(AppState targetState,
IOException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
String msg = "Unable to obtain user name, user not authenticated";
throw new AuthorizationException(msg);
@ -771,9 +784,14 @@ private RMApp getRMAppForAppId(String appId) {
}
private UserGroupInformation getCallerUserGroupInformation(
HttpServletRequest hsr) {
HttpServletRequest hsr, boolean usePrincipal) {
String remoteUser = hsr.getRemoteUser();
if (usePrincipal) {
Principal princ = hsr.getUserPrincipal();
remoteUser = princ == null ? null : princ.getName();
}
UserGroupInformation callerUGI = null;
if (remoteUser != null) {
callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
@ -799,7 +817,7 @@ private UserGroupInformation getCallerUserGroupInformation(
public Response createNewApplication(@Context HttpServletRequest hsr)
throws AuthorizationException, IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
throw new AuthorizationException("Unable to obtain user name, "
+ "user not authenticated");
@ -835,7 +853,7 @@ public Response submitApplication(ApplicationSubmissionContextInfo newApp,
IOException, InterruptedException {
init();
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
throw new AuthorizationException("Unable to obtain user name, "
+ "user not authenticated");
@ -887,8 +905,8 @@ private NewApplication createNewApplication() {
throw new YarnRuntimeException(msg, e);
}
NewApplication appId =
new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(
resp.getMaximumResourceCapability()));
new NewApplication(resp.getApplicationId().toString(),
new ResourceInfo(resp.getMaximumResourceCapability()));
return appId;
}
@ -962,7 +980,8 @@ protected Resource createAppSubmissionContextResource(
* @throws IOException
*/
protected ContainerLaunchContext createContainerLaunchContext(
ApplicationSubmissionContextInfo newApp) throws BadRequestException, IOException {
ApplicationSubmissionContextInfo newApp) throws BadRequestException,
IOException {
// create container launch context
@ -1033,4 +1052,238 @@ private Credentials createCredentials(CredentialsInfo credentials) {
}
return ret;
}
private UserGroupInformation createKerberosUserGroupInformation(
HttpServletRequest hsr) throws AuthorizationException, YarnException {
UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
if (callerUGI == null) {
String msg = "Unable to obtain user name, user not authenticated";
throw new AuthorizationException(msg);
}
String authType = hsr.getAuthType();
if (!KerberosAuthenticationHandler.TYPE.equals(authType)) {
String msg =
"Delegation token operations can only be carried out on a "
+ "Kerberos authenticated channel";
throw new YarnException(msg);
}
callerUGI.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
return callerUGI;
}
@POST
@Path("/delegation-token")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Response postDelegationToken(DelegationToken tokenData,
@Context HttpServletRequest hsr) throws AuthorizationException,
IOException, InterruptedException, Exception {
init();
UserGroupInformation callerUGI;
try {
callerUGI = createKerberosUserGroupInformation(hsr);
} catch (YarnException ye) {
return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
}
return createDelegationToken(tokenData, hsr, callerUGI);
}
@POST
@Path("/delegation-token/expiration")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
@Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Response
postDelegationTokenExpiration(@Context HttpServletRequest hsr)
throws AuthorizationException, IOException, InterruptedException,
Exception {
init();
UserGroupInformation callerUGI;
try {
callerUGI = createKerberosUserGroupInformation(hsr);
} catch (YarnException ye) {
return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
}
DelegationToken requestToken = new DelegationToken();
requestToken.setToken(extractToken(hsr).encodeToUrlString());
return renewDelegationToken(requestToken, hsr, callerUGI);
}
private Response createDelegationToken(DelegationToken tokenData,
HttpServletRequest hsr, UserGroupInformation callerUGI)
throws AuthorizationException, IOException, InterruptedException,
Exception {
final String renewer = tokenData.getRenewer();
GetDelegationTokenResponse resp;
try {
resp =
callerUGI
.doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {
@Override
public GetDelegationTokenResponse run() throws IOException,
YarnException {
GetDelegationTokenRequest createReq =
GetDelegationTokenRequest.newInstance(renewer);
return rm.getClientRMService().getDelegationToken(createReq);
}
});
} catch (Exception e) {
LOG.info("Create delegation token request failed", e);
throw e;
}
Token<RMDelegationTokenIdentifier> tk =
new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken()
.getIdentifier().array(), resp.getRMDelegationToken().getPassword()
.array(), new Text(resp.getRMDelegationToken().getKind()), new Text(
resp.getRMDelegationToken().getService()));
RMDelegationTokenIdentifier identifier = tk.decodeIdentifier();
long currentExpiration =
rm.getRMContext().getRMDelegationTokenSecretManager()
.getRenewDate(identifier);
DelegationToken respToken =
new DelegationToken(tk.encodeToUrlString(), renewer, identifier
.getOwner().toString(), tk.getKind().toString(), currentExpiration,
identifier.getMaxDate());
return Response.status(Status.OK).entity(respToken).build();
}
private Response renewDelegationToken(DelegationToken tokenData,
HttpServletRequest hsr, UserGroupInformation callerUGI)
throws AuthorizationException, IOException, InterruptedException,
Exception {
Token<RMDelegationTokenIdentifier> token =
extractToken(tokenData.getToken());
org.apache.hadoop.yarn.api.records.Token dToken =
BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
.toString(), token.getPassword(), token.getService().toString());
final RenewDelegationTokenRequest req =
RenewDelegationTokenRequest.newInstance(dToken);
RenewDelegationTokenResponse resp;
try {
resp =
callerUGI
.doAs(new PrivilegedExceptionAction<RenewDelegationTokenResponse>() {
@Override
public RenewDelegationTokenResponse run() throws IOException,
YarnException {
return rm.getClientRMService().renewDelegationToken(req);
}
});
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
if (ue.getCause().getCause() instanceof InvalidToken) {
throw new BadRequestException(ue.getCause().getCause().getMessage());
} else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
return Response.status(Status.FORBIDDEN)
.entity(ue.getCause().getCause().getMessage()).build();
}
LOG.info("Renew delegation token request failed", ue);
throw ue;
}
LOG.info("Renew delegation token request failed", ue);
throw ue;
} catch (Exception e) {
LOG.info("Renew delegation token request failed", e);
throw e;
}
long renewTime = resp.getNextExpirationTime();
DelegationToken respToken = new DelegationToken();
respToken.setNextExpirationTime(renewTime);
return Response.status(Status.OK).entity(respToken).build();
}
// For cancelling tokens, the encoded token is passed as a header
// There are two reasons for this -
// 1. Passing a request body as part of a DELETE request is not
// allowed by Jetty
// 2. Passing the encoded token as part of the url is not ideal
// since urls tend to get logged and anyone with access to
// the logs can extract tokens which are meant to be secret
@DELETE
@Path("/delegation-token")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public Response cancelDelegationToken(@Context HttpServletRequest hsr)
throws AuthorizationException, IOException, InterruptedException,
Exception {
init();
UserGroupInformation callerUGI;
try {
callerUGI = createKerberosUserGroupInformation(hsr);
} catch (YarnException ye) {
return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
}
Token<RMDelegationTokenIdentifier> token = extractToken(hsr);
org.apache.hadoop.yarn.api.records.Token dToken =
BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
.toString(), token.getPassword(), token.getService().toString());
final CancelDelegationTokenRequest req =
CancelDelegationTokenRequest.newInstance(dToken);
try {
callerUGI
.doAs(new PrivilegedExceptionAction<CancelDelegationTokenResponse>() {
@Override
public CancelDelegationTokenResponse run() throws IOException,
YarnException {
return rm.getClientRMService().cancelDelegationToken(req);
}
});
} catch (UndeclaredThrowableException ue) {
if (ue.getCause() instanceof YarnException) {
if (ue.getCause().getCause() instanceof InvalidToken) {
throw new BadRequestException(ue.getCause().getCause().getMessage());
} else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
return Response.status(Status.FORBIDDEN)
.entity(ue.getCause().getCause().getMessage()).build();
}
LOG.info("Renew delegation token request failed", ue);
throw ue;
}
LOG.info("Renew delegation token request failed", ue);
throw ue;
} catch (Exception e) {
LOG.info("Renew delegation token request failed", e);
throw e;
}
return Response.status(Status.OK).build();
}
private Token<RMDelegationTokenIdentifier> extractToken(
HttpServletRequest request) {
String encodedToken = request.getHeader(DELEGATION_TOKEN_HEADER);
if (encodedToken == null) {
String msg =
"Header '" + DELEGATION_TOKEN_HEADER
+ "' containing encoded token not found";
throw new BadRequestException(msg);
}
return extractToken(encodedToken);
}
private Token<RMDelegationTokenIdentifier> extractToken(String encodedToken) {
Token<RMDelegationTokenIdentifier> token =
new Token<RMDelegationTokenIdentifier>();
try {
token.decodeFromUrlString(encodedToken);
} catch (Exception ie) {
String msg = "Could not decode encoded token";
throw new BadRequestException(msg);
}
return token;
}
}

99
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/DelegationToken.java Normal file
View File

@ -0,0 +1,99 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@XmlRootElement(name = "delegation-token")
@XmlAccessorType(XmlAccessType.FIELD)
public class DelegationToken {
String token;
String renewer;
String owner;
String kind;
@XmlElement(name = "expiration-time")
Long nextExpirationTime;
@XmlElement(name = "max-validity")
Long maxValidity;
public DelegationToken() {
}
public DelegationToken(String token, String renewer, String owner,
String kind, Long nextExpirationTime, Long maxValidity) {
this.token = token;
this.renewer = renewer;
this.owner = owner;
this.kind = kind;
this.nextExpirationTime = nextExpirationTime;
this.maxValidity = maxValidity;
}
public String getToken() {
return token;
}
public String getRenewer() {
return renewer;
}
public Long getNextExpirationTime() {
return nextExpirationTime;
}
public void setToken(String token) {
this.token = token;
}
public void setRenewer(String renewer) {
this.renewer = renewer;
}
public void setNextExpirationTime(long nextExpirationTime) {
this.nextExpirationTime = Long.valueOf(nextExpirationTime);
}
public String getOwner() {
return owner;
}
public String getKind() {
return kind;
}
public Long getMaxValidity() {
return maxValidity;
}
public void setOwner(String owner) {
this.owner = owner;
}
public void setKind(String kind) {
this.kind = kind;
}
public void setMaxValidity(Long maxValidity) {
this.maxValidity = maxValidity;
}
}

784
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java Normal file
View File

@ -0,0 +1,784 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.Callable;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.authentication.KerberosTestUtils;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import org.w3c.dom.Document;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import com.google.inject.Guice;
import com.google.inject.Injector;
import com.google.inject.Singleton;
import com.google.inject.servlet.GuiceServletContextListener;
import com.google.inject.servlet.ServletModule;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
import com.sun.jersey.api.client.filter.LoggingFilter;
import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
import com.sun.jersey.test.framework.JerseyTest;
import com.sun.jersey.test.framework.WebAppDescriptor;
@RunWith(Parameterized.class)
public class TestRMWebServicesDelegationTokens extends JerseyTest {
private static final File testRootDir = new File("target",
TestRMWebServicesDelegationTokens.class.getName() + "-root");
private static File httpSpnegoKeytabFile = new File(
KerberosTestUtils.getKeytabFile());
private static String httpSpnegoPrincipal = KerberosTestUtils
.getServerPrincipal();
private static boolean miniKDCStarted = false;
private static MiniKdc testMiniKDC;
static {
try {
testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
} catch (Exception e) {
assertTrue("Couldn't create MiniKDC", false);
}
}
private static MockRM rm;
private Injector injector;
private boolean isKerberosAuth = false;
// Make sure the test uses the published header string
final String yarnTokenHeader = "Hadoop-YARN-RM-Delegation-Token";
@Singleton
public static class TestKerberosAuthFilter extends AuthenticationFilter {
@Override
protected Properties getConfiguration(String configPrefix,
FilterConfig filterConfig) throws ServletException {
Properties properties =
super.getConfiguration(configPrefix, filterConfig);
properties.put(KerberosAuthenticationHandler.PRINCIPAL,
httpSpnegoPrincipal);
properties.put(KerberosAuthenticationHandler.KEYTAB,
httpSpnegoKeytabFile.getAbsolutePath());
properties.put(AuthenticationFilter.AUTH_TYPE, "kerberos");
return properties;
}
}
@Singleton
public static class TestSimpleAuthFilter extends AuthenticationFilter {
@Override
protected Properties getConfiguration(String configPrefix,
FilterConfig filterConfig) throws ServletException {
Properties properties =
super.getConfiguration(configPrefix, filterConfig);
properties.put(KerberosAuthenticationHandler.PRINCIPAL,
httpSpnegoPrincipal);
properties.put(KerberosAuthenticationHandler.KEYTAB,
httpSpnegoKeytabFile.getAbsolutePath());
properties.put(AuthenticationFilter.AUTH_TYPE, "simple");
properties.put(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "false");
return properties;
}
}
private class TestServletModule extends ServletModule {
public Configuration rmconf = new Configuration();
@Override
protected void configureServlets() {
bind(JAXBContextResolver.class);
bind(RMWebServices.class);
bind(GenericExceptionHandler.class);
Configuration rmconf = new Configuration();
rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
ResourceScheduler.class);
rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
rm = new MockRM(rmconf);
bind(ResourceManager.class).toInstance(rm);
bind(RMContext.class).toInstance(rm.getRMContext());
bind(ApplicationACLsManager.class).toInstance(
rm.getApplicationACLsManager());
bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager());
if (isKerberosAuth == true) {
filter("/*").through(TestKerberosAuthFilter.class);
} else {
filter("/*").through(TestSimpleAuthFilter.class);
}
serve("/*").with(GuiceContainer.class);
}
}
private Injector getSimpleAuthInjector() {
return Guice.createInjector(new TestServletModule() {
@Override
protected void configureServlets() {
isKerberosAuth = false;
rmconf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"simple");
super.configureServlets();
}
});
}
private Injector getKerberosAuthInjector() {
return Guice.createInjector(new TestServletModule() {
@Override
protected void configureServlets() {
isKerberosAuth = true;
rmconf.set(
CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
httpSpnegoPrincipal);
rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
httpSpnegoKeytabFile.getAbsolutePath());
rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY,
httpSpnegoPrincipal);
rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
httpSpnegoKeytabFile.getAbsolutePath());
super.configureServlets();
}
});
}
public class GuiceServletConfig extends GuiceServletContextListener {
@Override
protected Injector getInjector() {
return injector;
}
}
@Parameters
public static Collection<Object[]> guiceConfigs() {
return Arrays.asList(new Object[][] { { 0 }, { 1 } });
}
public TestRMWebServicesDelegationTokens(int run) throws Exception {
super(new WebAppDescriptor.Builder(
"org.apache.hadoop.yarn.server.resourcemanager.webapp")
.contextListenerClass(GuiceServletConfig.class)
.filterClass(com.google.inject.servlet.GuiceFilter.class)
.contextPath("jersey-guice-filter").servletPath("/").build());
setupKDC();
switch (run) {
case 0:
default:
injector = getKerberosAuthInjector();
break;
case 1:
injector = getSimpleAuthInjector();
break;
}
}
private void setupKDC() throws Exception {
if (miniKDCStarted == false) {
testMiniKDC.start();
getKdc().createPrincipal(httpSpnegoKeytabFile, "HTTP/localhost",
"client", "client2", "client3");
miniKDCStarted = true;
}
}
private MiniKdc getKdc() {
return testMiniKDC;
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
httpSpnegoKeytabFile.deleteOnExit();
testRootDir.deleteOnExit();
}
@After
@Override
public void tearDown() throws Exception {
rm.stop();
super.tearDown();
}
// Simple test - try to create a delegation token via web services and check
// to make sure we get back a valid token. Validate token using RM function
// calls. It should only succeed with the kerberos filter
@Test
public void testCreateDelegationToken() throws Exception {
rm.start();
this.client().addFilter(new LoggingFilter(System.out));
final String renewer = "test-renewer";
String jsonBody = "{ \"renewer\" : \"" + renewer + "\" }";
String xmlBody =
"<delegation-token><renewer>" + renewer
+ "</renewer></delegation-token>";
String[] mediaTypes =
{ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
Map<String, String> bodyMap = new HashMap<String, String>();
bodyMap.put(MediaType.APPLICATION_JSON, jsonBody);
bodyMap.put(MediaType.APPLICATION_XML, xmlBody);
for (final String mediaType : mediaTypes) {
final String body = bodyMap.get(mediaType);
for (final String contentType : mediaTypes) {
if (isKerberosAuth == true) {
verifyKerberosAuthCreate(mediaType, contentType, body, renewer);
} else {
verifySimpleAuthCreate(mediaType, contentType, body);
}
}
}
rm.stop();
return;
}
private void verifySimpleAuthCreate(String mediaType, String contentType,
String body) {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").queryParam("user.name", "testuser")
.accept(contentType).entity(body, mediaType)
.post(ClientResponse.class);
assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
}
private void verifyKerberosAuthCreate(String mType, String cType,
String reqBody, String renUser) throws Exception {
final String mediaType = mType;
final String contentType = cType;
final String body = reqBody;
final String renewer = renUser;
KerberosTestUtils.doAsClient(new Callable<Void>() {
@Override
public Void call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.entity(body, mediaType).post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
DelegationToken tok = getDelegationTokenFromResponse(response);
assertFalse(tok.getToken().isEmpty());
Token<RMDelegationTokenIdentifier> token =
new Token<RMDelegationTokenIdentifier>();
token.decodeFromUrlString(tok.getToken());
assertEquals(renewer, token.decodeIdentifier().getRenewer().toString());
assertValidRMToken(tok.getToken());
DelegationToken dtoken = new DelegationToken();
response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.entity(dtoken, mediaType).post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
tok = getDelegationTokenFromResponse(response);
assertFalse(tok.getToken().isEmpty());
token = new Token<RMDelegationTokenIdentifier>();
token.decodeFromUrlString(tok.getToken());
assertEquals("", token.decodeIdentifier().getRenewer().toString());
assertValidRMToken(tok.getToken());
return null;
}
});
}
// Test to verify renew functionality - create a token and then try to renew
// it. The renewer should succeed; owner and third user should fail
@Test
public void testRenewDelegationToken() throws Exception {
client().addFilter(new LoggingFilter(System.out));
rm.start();
final String renewer = "client2";
this.client().addFilter(new LoggingFilter(System.out));
final DelegationToken dummyToken = new DelegationToken();
dummyToken.setRenewer(renewer);
String[] mediaTypes =
{ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
for (final String mediaType : mediaTypes) {
for (final String contentType : mediaTypes) {
if (isKerberosAuth == false) {
verifySimpleAuthRenew(mediaType, contentType);
continue;
}
// test "client" and client2" trying to renew "client" token
final DelegationToken responseToken =
KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
@Override
public DelegationToken call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.entity(dummyToken, mediaType).post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
DelegationToken tok = getDelegationTokenFromResponse(response);
assertFalse(tok.getToken().isEmpty());
String body = generateRenewTokenBody(mediaType, tok.getToken());
response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").path("expiration")
.header(yarnTokenHeader, tok.getToken())
.accept(contentType).entity(body, mediaType)
.post(ClientResponse.class);
assertEquals(Status.FORBIDDEN,
response.getClientResponseStatus());
return tok;
}
});
KerberosTestUtils.doAs(renewer, new Callable<DelegationToken>() {
@Override
public DelegationToken call() throws Exception {
// renew twice so that we can confirm that the
// expiration time actually changes
long oldExpirationTime = Time.now();
assertValidRMToken(responseToken.getToken());
String body =
generateRenewTokenBody(mediaType, responseToken.getToken());
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").path("expiration")
.header(yarnTokenHeader, responseToken.getToken())
.accept(contentType).entity(body, mediaType)
.post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
DelegationToken tok = getDelegationTokenFromResponse(response);
String message =
"Expiration time not as expected: old = " + oldExpirationTime
+ "; new = " + tok.getNextExpirationTime();
assertTrue(message, tok.getNextExpirationTime() > oldExpirationTime);
oldExpirationTime = tok.getNextExpirationTime();
// artificial sleep to ensure we get a different expiration time
Thread.sleep(1000);
response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").path("expiration")
.header(yarnTokenHeader, responseToken.getToken())
.accept(contentType).entity(body, mediaType)
.post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
tok = getDelegationTokenFromResponse(response);
message =
"Expiration time not as expected: old = " + oldExpirationTime
+ "; new = " + tok.getNextExpirationTime();
assertTrue(message, tok.getNextExpirationTime() > oldExpirationTime);
return tok;
}
});
// test unauthorized user renew attempt
KerberosTestUtils.doAs("client3", new Callable<DelegationToken>() {
@Override
public DelegationToken call() throws Exception {
String body =
generateRenewTokenBody(mediaType, responseToken.getToken());
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").path("expiration")
.header(yarnTokenHeader, responseToken.getToken())
.accept(contentType).entity(body, mediaType)
.post(ClientResponse.class);
assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
return null;
}
});
// test bad request - incorrect format, empty token string and random
// token string
KerberosTestUtils.doAsClient(new Callable<Void>() {
@Override
public Void call() throws Exception {
String token = "TEST_TOKEN_STRING";
String body = "";
if (mediaType.equals(MediaType.APPLICATION_JSON)) {
body = "{\"token\": \"" + token + "\" }";
} else {
body =
"<delegation-token><token>" + token
+ "</token></delegation-token>";
}
// missing token header
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").path("expiration")
.accept(contentType).entity(body, mediaType)
.post(ClientResponse.class);
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
return null;
}
});
}
}
rm.stop();
return;
}
private void verifySimpleAuthRenew(String mediaType, String contentType) {
String token = "TEST_TOKEN_STRING";
String body = "";
// contents of body don't matter because the request processing shouldn't
// get that far
if (mediaType.equals(MediaType.APPLICATION_JSON)) {
body = "{\"token\": \"" + token + "\" }";
body = "{\"abcd\": \"test-123\" }";
} else {
body =
"<delegation-token><token>" + token + "</token></delegation-token>";
body = "<delegation-token><xml>abcd</xml></delegation-token>";
}
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").queryParam("user.name", "testuser")
.accept(contentType).entity(body, mediaType)
.post(ClientResponse.class);
assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
}
// Test to verify cancel functionality - create a token and then try to cancel
// it. The owner and renewer should succeed; third user should fail
@Test
public void testCancelDelegationToken() throws Exception {
rm.start();
this.client().addFilter(new LoggingFilter(System.out));
if (isKerberosAuth == false) {
verifySimpleAuthCancel();
return;
}
final DelegationToken dtoken = new DelegationToken();
String renewer = "client2";
dtoken.setRenewer(renewer);
String[] mediaTypes =
{ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
for (final String mediaType : mediaTypes) {
for (final String contentType : mediaTypes) {
// owner should be able to cancel delegation token
KerberosTestUtils.doAsClient(new Callable<Void>() {
@Override
public Void call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.entity(dtoken, mediaType).post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
DelegationToken tok = getDelegationTokenFromResponse(response);
response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token")
.header(yarnTokenHeader, tok.getToken()).accept(contentType)
.delete(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
assertTokenCancelled(tok.getToken());
return null;
}
});
// renewer should be able to cancel token
final DelegationToken tmpToken =
KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
@Override
public DelegationToken call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.entity(dtoken, mediaType).post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
DelegationToken tok = getDelegationTokenFromResponse(response);
return tok;
}
});
KerberosTestUtils.doAs(renewer, new Callable<Void>() {
@Override
public Void call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token")
.header(yarnTokenHeader, tmpToken.getToken())
.accept(contentType).delete(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
assertTokenCancelled(tmpToken.getToken());
return null;
}
});
// third user should not be able to cancel token
final DelegationToken tmpToken2 =
KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
@Override
public DelegationToken call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.entity(dtoken, mediaType).post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
DelegationToken tok = getDelegationTokenFromResponse(response);
return tok;
}
});
KerberosTestUtils.doAs("client3", new Callable<Void>() {
@Override
public Void call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token")
.header(yarnTokenHeader, tmpToken2.getToken())
.accept(contentType).delete(ClientResponse.class);
assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
assertValidRMToken(tmpToken2.getToken());
return null;
}
});
testCancelTokenBadRequests(mediaType, contentType);
}
}
rm.stop();
return;
}
private void testCancelTokenBadRequests(String mType, String cType)
throws Exception {
final String mediaType = mType;
final String contentType = cType;
final DelegationToken dtoken = new DelegationToken();
String renewer = "client2";
dtoken.setRenewer(renewer);
// bad request(invalid header value)
KerberosTestUtils.doAsClient(new Callable<Void>() {
@Override
public Void call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token")
.header(yarnTokenHeader, "random-string").accept(contentType)
.delete(ClientResponse.class);
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
return null;
}
});
// bad request(missing header)
KerberosTestUtils.doAsClient(new Callable<Void>() {
@Override
public Void call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.delete(ClientResponse.class);
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
return null;
}
});
// bad request(cancelled token)
final DelegationToken tmpToken =
KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
@Override
public DelegationToken call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").accept(contentType)
.entity(dtoken, mediaType).post(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
DelegationToken tok = getDelegationTokenFromResponse(response);
return tok;
}
});
KerberosTestUtils.doAs(renewer, new Callable<Void>() {
@Override
public Void call() throws Exception {
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token")
.header(yarnTokenHeader, tmpToken.getToken()).accept(contentType)
.delete(ClientResponse.class);
assertEquals(Status.OK, response.getClientResponseStatus());
response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token")
.header(yarnTokenHeader, tmpToken.getToken()).accept(contentType)
.delete(ClientResponse.class);
assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
return null;
}
});
}
private void verifySimpleAuthCancel() {
// contents of header don't matter; request should never get that far
ClientResponse response =
resource().path("ws").path("v1").path("cluster")
.path("delegation-token").queryParam("user.name", "testuser")
.header(RMWebServices.DELEGATION_TOKEN_HEADER, "random")
.delete(ClientResponse.class);
assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
}
private DelegationToken
getDelegationTokenFromResponse(ClientResponse response)
throws IOException, ParserConfigurationException, SAXException,
JSONException {
if (response.getType().toString().equals(MediaType.APPLICATION_JSON)) {
return getDelegationTokenFromJson(response.getEntity(JSONObject.class));
}
return getDelegationTokenFromXML(response.getEntity(String.class));
}
public static DelegationToken getDelegationTokenFromXML(String tokenXML)
throws IOException, ParserConfigurationException, SAXException {
DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
DocumentBuilder db = dbf.newDocumentBuilder();
InputSource is = new InputSource();
is.setCharacterStream(new StringReader(tokenXML));
Document dom = db.parse(is);
NodeList nodes = dom.getElementsByTagName("delegation-token");
assertEquals("incorrect number of elements", 1, nodes.getLength());
Element element = (Element) nodes.item(0);
DelegationToken ret = new DelegationToken();
String token = WebServicesTestUtils.getXmlString(element, "token");
if (token != null) {
ret.setToken(token);
} else {
long expiration =
WebServicesTestUtils.getXmlLong(element, "expiration-time");
ret.setNextExpirationTime(expiration);
}
return ret;
}
public static DelegationToken getDelegationTokenFromJson(JSONObject json)
throws JSONException {
DelegationToken ret = new DelegationToken();
if (json.has("token")) {
ret.setToken(json.getString("token"));
} else if (json.has("expiration-time")) {
ret.setNextExpirationTime(json.getLong("expiration-time"));
}
return ret;
}
private void assertValidRMToken(String encodedToken) throws IOException {
Token<RMDelegationTokenIdentifier> realToken =
new Token<RMDelegationTokenIdentifier>();
realToken.decodeFromUrlString(encodedToken);
RMDelegationTokenIdentifier ident = realToken.decodeIdentifier();
rm.getRMContext().getRMDelegationTokenSecretManager()
.verifyToken(ident, realToken.getPassword());
assertTrue(rm.getRMContext().getRMDelegationTokenSecretManager()
.getAllTokens().containsKey(ident));
}
private void assertTokenCancelled(String encodedToken) throws Exception {
Token<RMDelegationTokenIdentifier> realToken =
new Token<RMDelegationTokenIdentifier>();
realToken.decodeFromUrlString(encodedToken);
RMDelegationTokenIdentifier ident = realToken.decodeIdentifier();
boolean exceptionCaught = false;
try {
rm.getRMContext().getRMDelegationTokenSecretManager()
.verifyToken(ident, realToken.getPassword());
} catch (InvalidToken it) {
exceptionCaught = true;
}
assertTrue("InvalidToken exception not thrown", exceptionCaught);
assertFalse(rm.getRMContext().getRMDelegationTokenSecretManager()
.getAllTokens().containsKey(ident));
}
private static String generateRenewTokenBody(String mediaType, String token) {
String body = "";
if (mediaType.equals(MediaType.APPLICATION_JSON)) {
body = "{\"token\": \"" + token + "\" }";
} else {
body =
"<delegation-token><token>" + token + "</token></delegation-token>";
}
return body;
}
}

220
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
View File

@ -2707,3 +2707,223 @@ Server: Jetty(6.1.26)
+---+
* Cluster {Delegation Tokens API}
The Delegation Tokens API can be used to create, renew and cancel YARN ResourceManager delegation tokens. All delegation token requests must be carried out on a Kerberos authenticated connection(using SPNEGO). Carrying out operations on a non-kerberos connection will result in a FORBIDDEN response. In case of renewing a token, only the renewer specified when creating the token can renew the token. Other users(including the owner) are forbidden from renewing tokens. It should be noted that when cancelling or renewing a token, the token to be cancelled or renewed is specified by setting a header.
This feature is currently in the alpha stage and may change in the future.
** URI
Use the following URI to create and cancel delegation tokens.
------
* http://<rm http address:port>/ws/v1/cluster/delegation-token
------
Use the following URI to renew delegation tokens.
------
* http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
------
** HTTP Operations Supported
------
* POST
* DELETE
------
** Query Parameters Supported
------
None
------
** Elements of the <delegation-token> object
The response from the delegation tokens API contains one of the fields listed below.
*---------------+--------------+-------------------------------+
|| Item || Data Type || Description |
*---------------+--------------+-------------------------------+
| token | string | The delegation token |
*---------------+--------------+-------------------------------+
| renewer | string | The user who is allowed to renew the delegation token |
*---------------+--------------+-------------------------------+
| owner | string | The owner of the delegation token |
*---------------+--------------+-------------------------------+
| kind | string | The kind of delegation token |
*---------------+--------------+-------------------------------+
| expiration-time | long | The expiration time of the token |
*---------------+--------------+-------------------------------+
| max-validity | long | The maximum validity of the token |
*---------------+--------------+-------------------------------+
** Response Examples
*** Creating a token
<<JSON response>>
HTTP Request:
------
POST http://<rm http address:port>/ws/v1/cluster/delegation-token
Accept: application/json
Content-Type: application/json
{
"renewer" : "test-renewer"
}
------
Response Header
+---+
HTTP/1.1 200 OK
WWW-Authenticate: Negotiate ...
Date: Sat, 28 Jun 2014 18:08:11 GMT
Server: Jetty(6.1.26)
Set-Cookie: ...
Content-Type: application/json
+---+
Response body
+---+
{
"token":"MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckiEZpigFHSJTKaQECFN9EMM9BzfPoDxu572EVUpzqhnSGE1JNX0RFTEVHQVRJT05fVE9LRU4A",
"renewer":"test-renewer",
"owner":"client@EXAMPLE.COM",
"kind":"RM_DELEGATION_TOKEN",
"expiration-time":"1405153616489",
"max-validity":"1405672016489"
}
+---+
<<XML response>>
HTTP Request
------
POST http://<rm http address:port>/ws/v1/cluster/delegation-token
Accept: application/xml
Content-Type: application/xml
<delegation-token>
<renewer>test-renewer</renewer>
</delegation-token>
------
Response Header
+---+
HTTP/1.1 200 OK
WWW-Authenticate: Negotiate ...
Date: Sat, 28 Jun 2014 18:08:11 GMT
Content-Length: 423
Server: Jetty(6.1.26)
Set-Cookie: ...
Content-Type: application/xml
+---+
Response Body
+---+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<delegation-token>
<token>MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckgZ8yigFHSI4jMgcCFDTG8X6XFFn2udQngzSXQL8vWaKIE1JNX0RFTEVHQVRJT05fVE9LRU4A</token>
<renewer>test-renewer</renewer>
<owner>client@EXAMPLE.COM</owner>
<kind>RM_DELEGATION_TOKEN</kind>
<expiration-time>1405153180466</expiration-time>
<max-validity>1405671580466</max-validity>
</delegation-token>
+---+
*** Renewing a token
<<JSON response>>
HTTP Request:
------
POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
Accept: application/json
Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
Content-Type: application/json
------
Response Header
+---+
HTTP/1.1 200 OK
WWW-Authenticate: Negotiate ...
Date: Sat, 28 Jun 2014 18:08:11 GMT
Server: Jetty(6.1.26)
Set-Cookie: ...
Content-Type: application/json
+---+
Response body
+---+
{
"expiration-time":"1404112520402"
}
+---+
<<XML response>>
HTTP Request
------
POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
Accept: application/xml
Content-Type: application/xml
Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
------
Response Header
+---+
HTTP/1.1 200 OK
WWW-Authenticate: Negotiate ...
Date: Sat, 28 Jun 2014 18:08:11 GMT
Content-Length: 423
Server: Jetty(6.1.26)
Set-Cookie: ...
Content-Type: application/xml
+---+
Response Body
+---+
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<delegation-token>
<expiration-time>1404112520402</expiration-time>
</delegation-token>
+---+
*** Cancelling a token
HTTP Request
-----
DELETE http://<rm http address:port>/ws/v1/cluster/delegation-token
Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
Accept: application/xml
-----
Response Header
+---+
HTTP/1.1 200 OK
WWW-Authenticate: Negotiate ...
Date: Sun, 29 Jun 2014 07:25:18 GMT
Transfer-Encoding: chunked
Server: Jetty(6.1.26)
Set-Cookie: ...
Content-Type: application/xml
+---+
No response body.