YARN-2233. Implemented ResourceManager web-services to create, renew and cancel delegation tokens. Contributed by Varun Vasudev.

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/trunk@1610876 13f79535-47bb-0310-9956-ffa450edef68

hadoop-common-project/hadoop-auth/pom.xml

@@ -139,6 +139,17 @@
           <attach>true</attach>
         </configuration>
       </plugin>
+      <plugin>
+        <groupId>org.apache.maven.plugins</groupId>
+        <artifactId>maven-jar-plugin</artifactId>
+        <executions>
+          <execution>
+            <goals>
+              <goal>test-jar</goal>
+            </goals>
+          </execution>
+        </executions>
+      </plugin>
     </plugins>
   </build>
 

hadoop-yarn-project/CHANGES.txt

@@ -95,6 +95,9 @@ Release 2.5.0 - UNRELEASED
     YARN-1713. Added get-new-app and submit-app functionality to RM web services.
     (Varun Vasudev via vinodkv)
 
+    YARN-2233. Implemented ResourceManager web-services to create, renew and
+    cancel delegation tokens. (Varun Vasudev via vinodkv)
+
   IMPROVEMENTS
 
     YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml

@@ -192,6 +192,21 @@
       <scope>test</scope>
     </dependency>
 
+    <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-minikdc</artifactId>
+      <scope>test</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.hadoop</groupId>
+      <artifactId>hadoop-auth</artifactId>
+      <scope>test</scope>
+      <type>test-jar</type>
+      <version>${project.version}</version>
+    </dependency>
+
     <!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
     <dependency>
       <groupId>com.sun.jersey.jersey-test-framework</groupId>

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java

@@ -29,8 +29,10 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
 import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
 import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager;
 import org.apache.hadoop.security.token.delegation.DelegationKey;
+import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenSecretManager.DelegationTokenInformation;
 import org.apache.hadoop.util.ExitUtil;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -193,4 +195,14 @@ public class RMDelegationTokenSecretManager extends
       addPersistedDelegationToken(entry.getKey(), entry.getValue());
     }
   }
+
+  public long getRenewDate(RMDelegationTokenIdentifier ident)
+      throws InvalidToken {
+    DelegationTokenInformation info = currentTokens.get(ident);
+    if (info == null) {
+      throw new InvalidToken("token (" + ident.toString()
+          + ") can't be found in cache");
+    }
+    return info.getRenewDate();
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java

@@ -22,6 +22,7 @@ import java.io.IOException;
 import java.lang.reflect.UndeclaredThrowableException;
 import java.security.AccessControlException;
 import java.nio.ByteBuffer;
+import java.security.Principal;
 import java.security.PrivilegedExceptionAction;
 import java.util.Arrays;
 import java.util.Collection;
@@ -36,6 +37,7 @@ import java.util.concurrent.ConcurrentMap;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.Consumes;
+import javax.ws.rs.DELETE;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
@@ -57,6 +59,8 @@ import org.apache.hadoop.io.DataOutputBuffer;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
 import org.apache.hadoop.security.authorize.AuthorizationException;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
@@ -67,6 +71,13 @@ import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.KillApplicationResponse;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetDelegationTokenResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
@@ -85,6 +96,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
@@ -109,6 +121,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedule
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CredentialsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FairSchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LocalResourceInfo;
@@ -118,6 +131,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
 import org.apache.hadoop.yarn.util.ConverterUtils;
 import org.apache.hadoop.yarn.webapp.BadRequestException;
 import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -139,6 +153,9 @@ public class RMWebServices {
   private final Configuration conf;
   private @Context HttpServletResponse response;
 
+  public final static String DELEGATION_TOKEN_HEADER =
+      "Hadoop-YARN-RM-Delegation-Token";
+
   @Inject
   public RMWebServices(final ResourceManager rm, Configuration conf) {
     this.rm = rm;
@@ -147,11 +164,7 @@ public class RMWebServices {
 
   protected Boolean hasAccess(RMApp app, HttpServletRequest hsr) {
     // Check for the authorization.
-    String remoteUser = hsr.getRemoteUser();
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
-    UserGroupInformation callerUGI = null;
-    if (remoteUser != null) {
-      callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
-    }
     if (callerUGI != null
         && !(this.rm.getApplicationACLsManager().checkAccess(callerUGI,
               ApplicationAccessType.VIEW_APP, app.getUser(),
@@ -626,7 +639,7 @@ public class RMWebServices {
   public AppState getAppState(@Context HttpServletRequest hsr,
       @PathParam("appid") String appId) throws AuthorizationException {
     init();
-    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
     String userName = "";
     if (callerUGI != null) {
       userName = callerUGI.getUserName();
@@ -661,7 +674,7 @@ public class RMWebServices {
       IOException {
 
     init();
-    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
     if (callerUGI == null) {
       String msg = "Unable to obtain user name, user not authenticated";
       throw new AuthorizationException(msg);
@@ -771,9 +784,14 @@ public class RMWebServices {
   }
 
   private UserGroupInformation getCallerUserGroupInformation(
-      HttpServletRequest hsr) {
+      HttpServletRequest hsr, boolean usePrincipal) {
 
     String remoteUser = hsr.getRemoteUser();
+    if (usePrincipal) {
+      Principal princ = hsr.getUserPrincipal();
+      remoteUser = princ == null ? null : princ.getName();
+    }
+
     UserGroupInformation callerUGI = null;
     if (remoteUser != null) {
       callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
@@ -799,7 +817,7 @@ public class RMWebServices {
   public Response createNewApplication(@Context HttpServletRequest hsr)
       throws AuthorizationException, IOException, InterruptedException {
     init();
-    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
     if (callerUGI == null) {
       throw new AuthorizationException("Unable to obtain user name, "
           + "user not authenticated");
@@ -835,7 +853,7 @@ public class RMWebServices {
       IOException, InterruptedException {
 
     init();
-    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr);
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
     if (callerUGI == null) {
       throw new AuthorizationException("Unable to obtain user name, "
           + "user not authenticated");
@@ -887,8 +905,8 @@ public class RMWebServices {
       throw new YarnRuntimeException(msg, e);
     }
     NewApplication appId =
-        new NewApplication(resp.getApplicationId().toString(), new ResourceInfo(
+        new NewApplication(resp.getApplicationId().toString(),
-          resp.getMaximumResourceCapability()));
+          new ResourceInfo(resp.getMaximumResourceCapability()));
     return appId;
   }
 
@@ -962,7 +980,8 @@ public class RMWebServices {
    * @throws IOException
    */
   protected ContainerLaunchContext createContainerLaunchContext(
-      ApplicationSubmissionContextInfo newApp) throws BadRequestException, IOException {
+      ApplicationSubmissionContextInfo newApp) throws BadRequestException,
+      IOException {
 
     // create container launch context
 
@@ -1033,4 +1052,238 @@ public class RMWebServices {
     }
     return ret;
   }
+
+  private UserGroupInformation createKerberosUserGroupInformation(
+      HttpServletRequest hsr) throws AuthorizationException, YarnException {
+
+    UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
+    if (callerUGI == null) {
+      String msg = "Unable to obtain user name, user not authenticated";
+      throw new AuthorizationException(msg);
+    }
+
+    String authType = hsr.getAuthType();
+    if (!KerberosAuthenticationHandler.TYPE.equals(authType)) {
+      String msg =
+          "Delegation token operations can only be carried out on a "
+              + "Kerberos authenticated channel";
+      throw new YarnException(msg);
+    }
+
+    callerUGI.setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+    return callerUGI;
+  }
+
+  @POST
+  @Path("/delegation-token")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response postDelegationToken(DelegationToken tokenData,
+      @Context HttpServletRequest hsr) throws AuthorizationException,
+      IOException, InterruptedException, Exception {
+
+    init();
+    UserGroupInformation callerUGI;
+    try {
+      callerUGI = createKerberosUserGroupInformation(hsr);
+    } catch (YarnException ye) {
+      return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
+    }
+    return createDelegationToken(tokenData, hsr, callerUGI);
+  }
+
+  @POST
+  @Path("/delegation-token/expiration")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response
+      postDelegationTokenExpiration(@Context HttpServletRequest hsr)
+          throws AuthorizationException, IOException, InterruptedException,
+          Exception {
+
+    init();
+    UserGroupInformation callerUGI;
+    try {
+      callerUGI = createKerberosUserGroupInformation(hsr);
+    } catch (YarnException ye) {
+      return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
+    }
+
+    DelegationToken requestToken = new DelegationToken();
+    requestToken.setToken(extractToken(hsr).encodeToUrlString());
+    return renewDelegationToken(requestToken, hsr, callerUGI);
+  }
+
+  private Response createDelegationToken(DelegationToken tokenData,
+      HttpServletRequest hsr, UserGroupInformation callerUGI)
+      throws AuthorizationException, IOException, InterruptedException,
+      Exception {
+
+    final String renewer = tokenData.getRenewer();
+    GetDelegationTokenResponse resp;
+    try {
+      resp =
+          callerUGI
+            .doAs(new PrivilegedExceptionAction<GetDelegationTokenResponse>() {
+              @Override
+              public GetDelegationTokenResponse run() throws IOException,
+                  YarnException {
+                GetDelegationTokenRequest createReq =
+                    GetDelegationTokenRequest.newInstance(renewer);
+                return rm.getClientRMService().getDelegationToken(createReq);
+              }
+            });
+    } catch (Exception e) {
+      LOG.info("Create delegation token request failed", e);
+      throw e;
+    }
+
+    Token<RMDelegationTokenIdentifier> tk =
+        new Token<RMDelegationTokenIdentifier>(resp.getRMDelegationToken()
+          .getIdentifier().array(), resp.getRMDelegationToken().getPassword()
+          .array(), new Text(resp.getRMDelegationToken().getKind()), new Text(
+          resp.getRMDelegationToken().getService()));
+    RMDelegationTokenIdentifier identifier = tk.decodeIdentifier();
+    long currentExpiration =
+        rm.getRMContext().getRMDelegationTokenSecretManager()
+          .getRenewDate(identifier);
+    DelegationToken respToken =
+        new DelegationToken(tk.encodeToUrlString(), renewer, identifier
+          .getOwner().toString(), tk.getKind().toString(), currentExpiration,
+          identifier.getMaxDate());
+    return Response.status(Status.OK).entity(respToken).build();
+  }
+
+  private Response renewDelegationToken(DelegationToken tokenData,
+      HttpServletRequest hsr, UserGroupInformation callerUGI)
+      throws AuthorizationException, IOException, InterruptedException,
+      Exception {
+
+    Token<RMDelegationTokenIdentifier> token =
+        extractToken(tokenData.getToken());
+
+    org.apache.hadoop.yarn.api.records.Token dToken =
+        BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
+          .toString(), token.getPassword(), token.getService().toString());
+    final RenewDelegationTokenRequest req =
+        RenewDelegationTokenRequest.newInstance(dToken);
+
+    RenewDelegationTokenResponse resp;
+    try {
+      resp =
+          callerUGI
+            .doAs(new PrivilegedExceptionAction<RenewDelegationTokenResponse>() {
+              @Override
+              public RenewDelegationTokenResponse run() throws IOException,
+                  YarnException {
+                return rm.getClientRMService().renewDelegationToken(req);
+              }
+            });
+    } catch (UndeclaredThrowableException ue) {
+      if (ue.getCause() instanceof YarnException) {
+        if (ue.getCause().getCause() instanceof InvalidToken) {
+          throw new BadRequestException(ue.getCause().getCause().getMessage());
+        } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
+          return Response.status(Status.FORBIDDEN)
+            .entity(ue.getCause().getCause().getMessage()).build();
+        }
+        LOG.info("Renew delegation token request failed", ue);
+        throw ue;
+      }
+      LOG.info("Renew delegation token request failed", ue);
+      throw ue;
+    } catch (Exception e) {
+      LOG.info("Renew delegation token request failed", e);
+      throw e;
+    }
+    long renewTime = resp.getNextExpirationTime();
+
+    DelegationToken respToken = new DelegationToken();
+    respToken.setNextExpirationTime(renewTime);
+    return Response.status(Status.OK).entity(respToken).build();
+  }
+
+  // For cancelling tokens, the encoded token is passed as a header
+  // There are two reasons for this -
+  // 1. Passing a request body as part of a DELETE request is not
+  // allowed by Jetty
+  // 2. Passing the encoded token as part of the url is not ideal
+  // since urls tend to get logged and anyone with access to
+  // the logs can extract tokens which are meant to be secret
+  @DELETE
+  @Path("/delegation-token")
+  @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
+  public Response cancelDelegationToken(@Context HttpServletRequest hsr)
+      throws AuthorizationException, IOException, InterruptedException,
+      Exception {
+
+    init();
+    UserGroupInformation callerUGI;
+    try {
+      callerUGI = createKerberosUserGroupInformation(hsr);
+    } catch (YarnException ye) {
+      return Response.status(Status.FORBIDDEN).entity(ye.getMessage()).build();
+    }
+
+    Token<RMDelegationTokenIdentifier> token = extractToken(hsr);
+
+    org.apache.hadoop.yarn.api.records.Token dToken =
+        BuilderUtils.newDelegationToken(token.getIdentifier(), token.getKind()
+          .toString(), token.getPassword(), token.getService().toString());
+    final CancelDelegationTokenRequest req =
+        CancelDelegationTokenRequest.newInstance(dToken);
+
+    try {
+      callerUGI
+        .doAs(new PrivilegedExceptionAction<CancelDelegationTokenResponse>() {
+          @Override
+          public CancelDelegationTokenResponse run() throws IOException,
+              YarnException {
+            return rm.getClientRMService().cancelDelegationToken(req);
+          }
+        });
+    } catch (UndeclaredThrowableException ue) {
+      if (ue.getCause() instanceof YarnException) {
+        if (ue.getCause().getCause() instanceof InvalidToken) {
+          throw new BadRequestException(ue.getCause().getCause().getMessage());
+        } else if (ue.getCause().getCause() instanceof org.apache.hadoop.security.AccessControlException) {
+          return Response.status(Status.FORBIDDEN)
+            .entity(ue.getCause().getCause().getMessage()).build();
+        }
+        LOG.info("Renew delegation token request failed", ue);
+        throw ue;
+      }
+      LOG.info("Renew delegation token request failed", ue);
+      throw ue;
+    } catch (Exception e) {
+      LOG.info("Renew delegation token request failed", e);
+      throw e;
+    }
+
+    return Response.status(Status.OK).build();
+  }
+
+  private Token<RMDelegationTokenIdentifier> extractToken(
+      HttpServletRequest request) {
+    String encodedToken = request.getHeader(DELEGATION_TOKEN_HEADER);
+    if (encodedToken == null) {
+      String msg =
+          "Header '" + DELEGATION_TOKEN_HEADER
+              + "' containing encoded token not found";
+      throw new BadRequestException(msg);
+    }
+    return extractToken(encodedToken);
+  }
+
+  private Token<RMDelegationTokenIdentifier> extractToken(String encodedToken) {
+    Token<RMDelegationTokenIdentifier> token =
+        new Token<RMDelegationTokenIdentifier>();
+    try {
+      token.decodeFromUrlString(encodedToken);
+    } catch (Exception ie) {
+      String msg = "Could not decode encoded token";
+      throw new BadRequestException(msg);
+    }
+    return token;
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/DelegationToken.java

@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+@XmlRootElement(name = "delegation-token")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class DelegationToken {
+
+  String token;
+  String renewer;
+  String owner;
+  String kind;
+  @XmlElement(name = "expiration-time")
+  Long nextExpirationTime;
+  @XmlElement(name = "max-validity")
+  Long maxValidity;
+
+  public DelegationToken() {
+  }
+
+  public DelegationToken(String token, String renewer, String owner,
+      String kind, Long nextExpirationTime, Long maxValidity) {
+    this.token = token;
+    this.renewer = renewer;
+    this.owner = owner;
+    this.kind = kind;
+    this.nextExpirationTime = nextExpirationTime;
+    this.maxValidity = maxValidity;
+  }
+
+  public String getToken() {
+    return token;
+  }
+
+  public String getRenewer() {
+    return renewer;
+  }
+
+  public Long getNextExpirationTime() {
+    return nextExpirationTime;
+  }
+
+  public void setToken(String token) {
+    this.token = token;
+  }
+
+  public void setRenewer(String renewer) {
+    this.renewer = renewer;
+  }
+
+  public void setNextExpirationTime(long nextExpirationTime) {
+    this.nextExpirationTime = Long.valueOf(nextExpirationTime);
+  }
+
+  public String getOwner() {
+    return owner;
+  }
+
+  public String getKind() {
+    return kind;
+  }
+
+  public Long getMaxValidity() {
+    return maxValidity;
+  }
+
+  public void setOwner(String owner) {
+    this.owner = owner;
+  }
+
+  public void setKind(String kind) {
+    this.kind = kind;
+  }
+
+  public void setMaxValidity(Long maxValidity) {
+    this.maxValidity = maxValidity;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java

@@ -0,0 +1,784 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.Callable;
+
+import javax.servlet.FilterConfig;
+import javax.servlet.ServletException;
+import javax.ws.rs.core.MediaType;
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
+import javax.xml.parsers.ParserConfigurationException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.authentication.KerberosTestUtils;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import org.apache.hadoop.security.authentication.server.KerberosAuthenticationHandler;
+import org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+
+import com.google.inject.Guice;
+import com.google.inject.Injector;
+import com.google.inject.Singleton;
+import com.google.inject.servlet.GuiceServletContextListener;
+import com.google.inject.servlet.ServletModule;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.filter.LoggingFilter;
+import com.sun.jersey.guice.spi.container.servlet.GuiceContainer;
+import com.sun.jersey.test.framework.JerseyTest;
+import com.sun.jersey.test.framework.WebAppDescriptor;
+
+@RunWith(Parameterized.class)
+public class TestRMWebServicesDelegationTokens extends JerseyTest {
+
+  private static final File testRootDir = new File("target",
+    TestRMWebServicesDelegationTokens.class.getName() + "-root");
+  private static File httpSpnegoKeytabFile = new File(
+    KerberosTestUtils.getKeytabFile());
+
+  private static String httpSpnegoPrincipal = KerberosTestUtils
+    .getServerPrincipal();
+
+  private static boolean miniKDCStarted = false;
+  private static MiniKdc testMiniKDC;
+  static {
+    try {
+      testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
+    } catch (Exception e) {
+      assertTrue("Couldn't create MiniKDC", false);
+    }
+  }
+
+  private static MockRM rm;
+
+  private Injector injector;
+
+  private boolean isKerberosAuth = false;
+
+  // Make sure the test uses the published header string
+  final String yarnTokenHeader = "Hadoop-YARN-RM-Delegation-Token";
+
+  @Singleton
+  public static class TestKerberosAuthFilter extends AuthenticationFilter {
+    @Override
+    protected Properties getConfiguration(String configPrefix,
+        FilterConfig filterConfig) throws ServletException {
+
+      Properties properties =
+          super.getConfiguration(configPrefix, filterConfig);
+
+      properties.put(KerberosAuthenticationHandler.PRINCIPAL,
+        httpSpnegoPrincipal);
+      properties.put(KerberosAuthenticationHandler.KEYTAB,
+        httpSpnegoKeytabFile.getAbsolutePath());
+      properties.put(AuthenticationFilter.AUTH_TYPE, "kerberos");
+      return properties;
+    }
+  }
+
+  @Singleton
+  public static class TestSimpleAuthFilter extends AuthenticationFilter {
+    @Override
+    protected Properties getConfiguration(String configPrefix,
+        FilterConfig filterConfig) throws ServletException {
+
+      Properties properties =
+          super.getConfiguration(configPrefix, filterConfig);
+
+      properties.put(KerberosAuthenticationHandler.PRINCIPAL,
+        httpSpnegoPrincipal);
+      properties.put(KerberosAuthenticationHandler.KEYTAB,
+        httpSpnegoKeytabFile.getAbsolutePath());
+      properties.put(AuthenticationFilter.AUTH_TYPE, "simple");
+      properties.put(PseudoAuthenticationHandler.ANONYMOUS_ALLOWED, "false");
+      return properties;
+    }
+  }
+
+  private class TestServletModule extends ServletModule {
+    public Configuration rmconf = new Configuration();
+
+    @Override
+    protected void configureServlets() {
+      bind(JAXBContextResolver.class);
+      bind(RMWebServices.class);
+      bind(GenericExceptionHandler.class);
+      Configuration rmconf = new Configuration();
+      rmconf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+        YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
+      rmconf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
+        ResourceScheduler.class);
+      rmconf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
+      rm = new MockRM(rmconf);
+      bind(ResourceManager.class).toInstance(rm);
+      bind(RMContext.class).toInstance(rm.getRMContext());
+      bind(ApplicationACLsManager.class).toInstance(
+        rm.getApplicationACLsManager());
+      bind(QueueACLsManager.class).toInstance(rm.getQueueACLsManager());
+      if (isKerberosAuth == true) {
+        filter("/*").through(TestKerberosAuthFilter.class);
+      } else {
+        filter("/*").through(TestSimpleAuthFilter.class);
+      }
+      serve("/*").with(GuiceContainer.class);
+    }
+  }
+
+  private Injector getSimpleAuthInjector() {
+    return Guice.createInjector(new TestServletModule() {
+      @Override
+      protected void configureServlets() {
+        isKerberosAuth = false;
+        rmconf.set(
+          CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+          "simple");
+        super.configureServlets();
+      }
+    });
+  }
+
+  private Injector getKerberosAuthInjector() {
+    return Guice.createInjector(new TestServletModule() {
+      @Override
+      protected void configureServlets() {
+        isKerberosAuth = true;
+        rmconf.set(
+          CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+          "kerberos");
+        rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_USER_NAME_KEY,
+          httpSpnegoPrincipal);
+        rmconf.set(YarnConfiguration.RM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
+          httpSpnegoKeytabFile.getAbsolutePath());
+        rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_USER_NAME_KEY,
+          httpSpnegoPrincipal);
+        rmconf.set(YarnConfiguration.NM_WEBAPP_SPNEGO_KEYTAB_FILE_KEY,
+          httpSpnegoKeytabFile.getAbsolutePath());
+
+        super.configureServlets();
+      }
+    });
+  }
+
+  public class GuiceServletConfig extends GuiceServletContextListener {
+
+    @Override
+    protected Injector getInjector() {
+      return injector;
+    }
+  }
+
+  @Parameters
+  public static Collection<Object[]> guiceConfigs() {
+    return Arrays.asList(new Object[][] { { 0 }, { 1 } });
+  }
+
+  public TestRMWebServicesDelegationTokens(int run) throws Exception {
+    super(new WebAppDescriptor.Builder(
+      "org.apache.hadoop.yarn.server.resourcemanager.webapp")
+      .contextListenerClass(GuiceServletConfig.class)
+      .filterClass(com.google.inject.servlet.GuiceFilter.class)
+      .contextPath("jersey-guice-filter").servletPath("/").build());
+    setupKDC();
+    switch (run) {
+    case 0:
+    default:
+      injector = getKerberosAuthInjector();
+      break;
+    case 1:
+      injector = getSimpleAuthInjector();
+      break;
+    }
+  }
+
+  private void setupKDC() throws Exception {
+    if (miniKDCStarted == false) {
+      testMiniKDC.start();
+      getKdc().createPrincipal(httpSpnegoKeytabFile, "HTTP/localhost",
+        "client", "client2", "client3");
+      miniKDCStarted = true;
+    }
+  }
+
+  private MiniKdc getKdc() {
+    return testMiniKDC;
+  }
+
+  @Before
+  @Override
+  public void setUp() throws Exception {
+    super.setUp();
+    httpSpnegoKeytabFile.deleteOnExit();
+    testRootDir.deleteOnExit();
+  }
+
+  @After
+  @Override
+  public void tearDown() throws Exception {
+    rm.stop();
+    super.tearDown();
+  }
+
+  // Simple test - try to create a delegation token via web services and check
+  // to make sure we get back a valid token. Validate token using RM function
+  // calls. It should only succeed with the kerberos filter
+  @Test
+  public void testCreateDelegationToken() throws Exception {
+    rm.start();
+    this.client().addFilter(new LoggingFilter(System.out));
+    final String renewer = "test-renewer";
+    String jsonBody = "{ \"renewer\" : \"" + renewer + "\" }";
+    String xmlBody =
+        "<delegation-token><renewer>" + renewer
+            + "</renewer></delegation-token>";
+    String[] mediaTypes =
+        { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+    Map<String, String> bodyMap = new HashMap<String, String>();
+    bodyMap.put(MediaType.APPLICATION_JSON, jsonBody);
+    bodyMap.put(MediaType.APPLICATION_XML, xmlBody);
+    for (final String mediaType : mediaTypes) {
+      final String body = bodyMap.get(mediaType);
+      for (final String contentType : mediaTypes) {
+        if (isKerberosAuth == true) {
+          verifyKerberosAuthCreate(mediaType, contentType, body, renewer);
+        } else {
+          verifySimpleAuthCreate(mediaType, contentType, body);
+        }
+      }
+    }
+
+    rm.stop();
+    return;
+  }
+
+  private void verifySimpleAuthCreate(String mediaType, String contentType,
+      String body) {
+    ClientResponse response =
+        resource().path("ws").path("v1").path("cluster")
+          .path("delegation-token").queryParam("user.name", "testuser")
+          .accept(contentType).entity(body, mediaType)
+          .post(ClientResponse.class);
+    assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+  }
+
+  private void verifyKerberosAuthCreate(String mType, String cType,
+      String reqBody, String renUser) throws Exception {
+    final String mediaType = mType;
+    final String contentType = cType;
+    final String body = reqBody;
+    final String renewer = renUser;
+    KerberosTestUtils.doAsClient(new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        ClientResponse response =
+            resource().path("ws").path("v1").path("cluster")
+              .path("delegation-token").accept(contentType)
+              .entity(body, mediaType).post(ClientResponse.class);
+        assertEquals(Status.OK, response.getClientResponseStatus());
+        DelegationToken tok = getDelegationTokenFromResponse(response);
+        assertFalse(tok.getToken().isEmpty());
+        Token<RMDelegationTokenIdentifier> token =
+            new Token<RMDelegationTokenIdentifier>();
+        token.decodeFromUrlString(tok.getToken());
+        assertEquals(renewer, token.decodeIdentifier().getRenewer().toString());
+        assertValidRMToken(tok.getToken());
+        DelegationToken dtoken = new DelegationToken();
+        response =
+            resource().path("ws").path("v1").path("cluster")
+              .path("delegation-token").accept(contentType)
+              .entity(dtoken, mediaType).post(ClientResponse.class);
+        assertEquals(Status.OK, response.getClientResponseStatus());
+        tok = getDelegationTokenFromResponse(response);
+        assertFalse(tok.getToken().isEmpty());
+        token = new Token<RMDelegationTokenIdentifier>();
+        token.decodeFromUrlString(tok.getToken());
+        assertEquals("", token.decodeIdentifier().getRenewer().toString());
+        assertValidRMToken(tok.getToken());
+        return null;
+      }
+    });
+  }
+
+  // Test to verify renew functionality - create a token and then try to renew
+  // it. The renewer should succeed; owner and third user should fail
+  @Test
+  public void testRenewDelegationToken() throws Exception {
+    client().addFilter(new LoggingFilter(System.out));
+    rm.start();
+    final String renewer = "client2";
+    this.client().addFilter(new LoggingFilter(System.out));
+    final DelegationToken dummyToken = new DelegationToken();
+    dummyToken.setRenewer(renewer);
+    String[] mediaTypes =
+        { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+    for (final String mediaType : mediaTypes) {
+      for (final String contentType : mediaTypes) {
+
+        if (isKerberosAuth == false) {
+          verifySimpleAuthRenew(mediaType, contentType);
+          continue;
+        }
+
+        // test "client" and client2" trying to renew "client" token
+        final DelegationToken responseToken =
+            KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
+              @Override
+              public DelegationToken call() throws Exception {
+                ClientResponse response =
+                    resource().path("ws").path("v1").path("cluster")
+                      .path("delegation-token").accept(contentType)
+                      .entity(dummyToken, mediaType).post(ClientResponse.class);
+                assertEquals(Status.OK, response.getClientResponseStatus());
+                DelegationToken tok = getDelegationTokenFromResponse(response);
+                assertFalse(tok.getToken().isEmpty());
+                String body = generateRenewTokenBody(mediaType, tok.getToken());
+                response =
+                    resource().path("ws").path("v1").path("cluster")
+                      .path("delegation-token").path("expiration")
+                      .header(yarnTokenHeader, tok.getToken())
+                      .accept(contentType).entity(body, mediaType)
+                      .post(ClientResponse.class);
+                assertEquals(Status.FORBIDDEN,
+                  response.getClientResponseStatus());
+                return tok;
+              }
+            });
+
+        KerberosTestUtils.doAs(renewer, new Callable<DelegationToken>() {
+          @Override
+          public DelegationToken call() throws Exception {
+            // renew twice so that we can confirm that the
+            // expiration time actually changes
+            long oldExpirationTime = Time.now();
+            assertValidRMToken(responseToken.getToken());
+            String body =
+                generateRenewTokenBody(mediaType, responseToken.getToken());
+            ClientResponse response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token").path("expiration")
+                  .header(yarnTokenHeader, responseToken.getToken())
+                  .accept(contentType).entity(body, mediaType)
+                  .post(ClientResponse.class);
+            assertEquals(Status.OK, response.getClientResponseStatus());
+            DelegationToken tok = getDelegationTokenFromResponse(response);
+            String message =
+                "Expiration time not as expected: old = " + oldExpirationTime
+                    + "; new = " + tok.getNextExpirationTime();
+            assertTrue(message, tok.getNextExpirationTime() > oldExpirationTime);
+            oldExpirationTime = tok.getNextExpirationTime();
+            // artificial sleep to ensure we get a different expiration time
+            Thread.sleep(1000);
+            response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token").path("expiration")
+                  .header(yarnTokenHeader, responseToken.getToken())
+                  .accept(contentType).entity(body, mediaType)
+                  .post(ClientResponse.class);
+            assertEquals(Status.OK, response.getClientResponseStatus());
+            tok = getDelegationTokenFromResponse(response);
+            message =
+                "Expiration time not as expected: old = " + oldExpirationTime
+                    + "; new = " + tok.getNextExpirationTime();
+            assertTrue(message, tok.getNextExpirationTime() > oldExpirationTime);
+            return tok;
+          }
+        });
+
+        // test unauthorized user renew attempt
+        KerberosTestUtils.doAs("client3", new Callable<DelegationToken>() {
+          @Override
+          public DelegationToken call() throws Exception {
+            String body =
+                generateRenewTokenBody(mediaType, responseToken.getToken());
+            ClientResponse response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token").path("expiration")
+                  .header(yarnTokenHeader, responseToken.getToken())
+                  .accept(contentType).entity(body, mediaType)
+                  .post(ClientResponse.class);
+            assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+            return null;
+          }
+        });
+
+        // test bad request - incorrect format, empty token string and random
+        // token string
+        KerberosTestUtils.doAsClient(new Callable<Void>() {
+          @Override
+          public Void call() throws Exception {
+            String token = "TEST_TOKEN_STRING";
+            String body = "";
+            if (mediaType.equals(MediaType.APPLICATION_JSON)) {
+              body = "{\"token\": \"" + token + "\" }";
+            } else {
+              body =
+                  "<delegation-token><token>" + token
+                      + "</token></delegation-token>";
+            }
+
+            // missing token header
+            ClientResponse response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token").path("expiration")
+                  .accept(contentType).entity(body, mediaType)
+                  .post(ClientResponse.class);
+            assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+            return null;
+          }
+        });
+      }
+    }
+
+    rm.stop();
+    return;
+  }
+
+  private void verifySimpleAuthRenew(String mediaType, String contentType) {
+    String token = "TEST_TOKEN_STRING";
+    String body = "";
+    // contents of body don't matter because the request processing shouldn't
+    // get that far
+    if (mediaType.equals(MediaType.APPLICATION_JSON)) {
+      body = "{\"token\": \"" + token + "\" }";
+      body = "{\"abcd\": \"test-123\" }";
+    } else {
+      body =
+          "<delegation-token><token>" + token + "</token></delegation-token>";
+      body = "<delegation-token><xml>abcd</xml></delegation-token>";
+    }
+    ClientResponse response =
+        resource().path("ws").path("v1").path("cluster")
+          .path("delegation-token").queryParam("user.name", "testuser")
+          .accept(contentType).entity(body, mediaType)
+          .post(ClientResponse.class);
+    assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+  }
+
+  // Test to verify cancel functionality - create a token and then try to cancel
+  // it. The owner and renewer should succeed; third user should fail
+  @Test
+  public void testCancelDelegationToken() throws Exception {
+    rm.start();
+    this.client().addFilter(new LoggingFilter(System.out));
+    if (isKerberosAuth == false) {
+      verifySimpleAuthCancel();
+      return;
+    }
+
+    final DelegationToken dtoken = new DelegationToken();
+    String renewer = "client2";
+    dtoken.setRenewer(renewer);
+    String[] mediaTypes =
+        { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
+    for (final String mediaType : mediaTypes) {
+      for (final String contentType : mediaTypes) {
+
+        // owner should be able to cancel delegation token
+        KerberosTestUtils.doAsClient(new Callable<Void>() {
+          @Override
+          public Void call() throws Exception {
+            ClientResponse response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token").accept(contentType)
+                  .entity(dtoken, mediaType).post(ClientResponse.class);
+            assertEquals(Status.OK, response.getClientResponseStatus());
+            DelegationToken tok = getDelegationTokenFromResponse(response);
+            response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token")
+                  .header(yarnTokenHeader, tok.getToken()).accept(contentType)
+                  .delete(ClientResponse.class);
+            assertEquals(Status.OK, response.getClientResponseStatus());
+            assertTokenCancelled(tok.getToken());
+            return null;
+          }
+        });
+
+        // renewer should be able to cancel token
+        final DelegationToken tmpToken =
+            KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
+              @Override
+              public DelegationToken call() throws Exception {
+                ClientResponse response =
+                    resource().path("ws").path("v1").path("cluster")
+                      .path("delegation-token").accept(contentType)
+                      .entity(dtoken, mediaType).post(ClientResponse.class);
+                assertEquals(Status.OK, response.getClientResponseStatus());
+                DelegationToken tok = getDelegationTokenFromResponse(response);
+                return tok;
+              }
+            });
+
+        KerberosTestUtils.doAs(renewer, new Callable<Void>() {
+          @Override
+          public Void call() throws Exception {
+            ClientResponse response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token")
+                  .header(yarnTokenHeader, tmpToken.getToken())
+                  .accept(contentType).delete(ClientResponse.class);
+            assertEquals(Status.OK, response.getClientResponseStatus());
+            assertTokenCancelled(tmpToken.getToken());
+            return null;
+          }
+        });
+
+        // third user should not be able to cancel token
+        final DelegationToken tmpToken2 =
+            KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
+              @Override
+              public DelegationToken call() throws Exception {
+                ClientResponse response =
+                    resource().path("ws").path("v1").path("cluster")
+                      .path("delegation-token").accept(contentType)
+                      .entity(dtoken, mediaType).post(ClientResponse.class);
+                assertEquals(Status.OK, response.getClientResponseStatus());
+                DelegationToken tok = getDelegationTokenFromResponse(response);
+                return tok;
+              }
+            });
+
+        KerberosTestUtils.doAs("client3", new Callable<Void>() {
+          @Override
+          public Void call() throws Exception {
+            ClientResponse response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token")
+                  .header(yarnTokenHeader, tmpToken2.getToken())
+                  .accept(contentType).delete(ClientResponse.class);
+            assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+            assertValidRMToken(tmpToken2.getToken());
+            return null;
+          }
+        });
+
+        testCancelTokenBadRequests(mediaType, contentType);
+      }
+    }
+
+    rm.stop();
+    return;
+  }
+
+  private void testCancelTokenBadRequests(String mType, String cType)
+      throws Exception {
+
+    final String mediaType = mType;
+    final String contentType = cType;
+    final DelegationToken dtoken = new DelegationToken();
+    String renewer = "client2";
+    dtoken.setRenewer(renewer);
+
+    // bad request(invalid header value)
+    KerberosTestUtils.doAsClient(new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        ClientResponse response =
+            resource().path("ws").path("v1").path("cluster")
+              .path("delegation-token")
+              .header(yarnTokenHeader, "random-string").accept(contentType)
+              .delete(ClientResponse.class);
+        assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+        return null;
+      }
+    });
+
+    // bad request(missing header)
+    KerberosTestUtils.doAsClient(new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        ClientResponse response =
+            resource().path("ws").path("v1").path("cluster")
+              .path("delegation-token").accept(contentType)
+              .delete(ClientResponse.class);
+        assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+
+        return null;
+      }
+    });
+
+    // bad request(cancelled token)
+    final DelegationToken tmpToken =
+        KerberosTestUtils.doAsClient(new Callable<DelegationToken>() {
+          @Override
+          public DelegationToken call() throws Exception {
+            ClientResponse response =
+                resource().path("ws").path("v1").path("cluster")
+                  .path("delegation-token").accept(contentType)
+                  .entity(dtoken, mediaType).post(ClientResponse.class);
+            assertEquals(Status.OK, response.getClientResponseStatus());
+            DelegationToken tok = getDelegationTokenFromResponse(response);
+            return tok;
+          }
+        });
+
+    KerberosTestUtils.doAs(renewer, new Callable<Void>() {
+      @Override
+      public Void call() throws Exception {
+        ClientResponse response =
+            resource().path("ws").path("v1").path("cluster")
+              .path("delegation-token")
+              .header(yarnTokenHeader, tmpToken.getToken()).accept(contentType)
+              .delete(ClientResponse.class);
+        assertEquals(Status.OK, response.getClientResponseStatus());
+        response =
+            resource().path("ws").path("v1").path("cluster")
+              .path("delegation-token")
+              .header(yarnTokenHeader, tmpToken.getToken()).accept(contentType)
+              .delete(ClientResponse.class);
+        assertEquals(Status.BAD_REQUEST, response.getClientResponseStatus());
+        return null;
+      }
+    });
+  }
+
+  private void verifySimpleAuthCancel() {
+    // contents of header don't matter; request should never get that far
+    ClientResponse response =
+        resource().path("ws").path("v1").path("cluster")
+          .path("delegation-token").queryParam("user.name", "testuser")
+          .header(RMWebServices.DELEGATION_TOKEN_HEADER, "random")
+          .delete(ClientResponse.class);
+    assertEquals(Status.FORBIDDEN, response.getClientResponseStatus());
+  }
+
+  private DelegationToken
+      getDelegationTokenFromResponse(ClientResponse response)
+          throws IOException, ParserConfigurationException, SAXException,
+          JSONException {
+    if (response.getType().toString().equals(MediaType.APPLICATION_JSON)) {
+      return getDelegationTokenFromJson(response.getEntity(JSONObject.class));
+    }
+    return getDelegationTokenFromXML(response.getEntity(String.class));
+  }
+
+  public static DelegationToken getDelegationTokenFromXML(String tokenXML)
+      throws IOException, ParserConfigurationException, SAXException {
+    DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
+    DocumentBuilder db = dbf.newDocumentBuilder();
+    InputSource is = new InputSource();
+    is.setCharacterStream(new StringReader(tokenXML));
+    Document dom = db.parse(is);
+    NodeList nodes = dom.getElementsByTagName("delegation-token");
+    assertEquals("incorrect number of elements", 1, nodes.getLength());
+    Element element = (Element) nodes.item(0);
+    DelegationToken ret = new DelegationToken();
+    String token = WebServicesTestUtils.getXmlString(element, "token");
+    if (token != null) {
+      ret.setToken(token);
+    } else {
+      long expiration =
+          WebServicesTestUtils.getXmlLong(element, "expiration-time");
+      ret.setNextExpirationTime(expiration);
+    }
+    return ret;
+  }
+
+  public static DelegationToken getDelegationTokenFromJson(JSONObject json)
+      throws JSONException {
+    DelegationToken ret = new DelegationToken();
+    if (json.has("token")) {
+      ret.setToken(json.getString("token"));
+    } else if (json.has("expiration-time")) {
+      ret.setNextExpirationTime(json.getLong("expiration-time"));
+    }
+    return ret;
+  }
+
+  private void assertValidRMToken(String encodedToken) throws IOException {
+    Token<RMDelegationTokenIdentifier> realToken =
+        new Token<RMDelegationTokenIdentifier>();
+    realToken.decodeFromUrlString(encodedToken);
+    RMDelegationTokenIdentifier ident = realToken.decodeIdentifier();
+    rm.getRMContext().getRMDelegationTokenSecretManager()
+      .verifyToken(ident, realToken.getPassword());
+    assertTrue(rm.getRMContext().getRMDelegationTokenSecretManager()
+      .getAllTokens().containsKey(ident));
+  }
+
+  private void assertTokenCancelled(String encodedToken) throws Exception {
+    Token<RMDelegationTokenIdentifier> realToken =
+        new Token<RMDelegationTokenIdentifier>();
+    realToken.decodeFromUrlString(encodedToken);
+    RMDelegationTokenIdentifier ident = realToken.decodeIdentifier();
+    boolean exceptionCaught = false;
+    try {
+      rm.getRMContext().getRMDelegationTokenSecretManager()
+        .verifyToken(ident, realToken.getPassword());
+    } catch (InvalidToken it) {
+      exceptionCaught = true;
+    }
+    assertTrue("InvalidToken exception not thrown", exceptionCaught);
+    assertFalse(rm.getRMContext().getRMDelegationTokenSecretManager()
+      .getAllTokens().containsKey(ident));
+  }
+
+  private static String generateRenewTokenBody(String mediaType, String token) {
+    String body = "";
+    if (mediaType.equals(MediaType.APPLICATION_JSON)) {
+      body = "{\"token\": \"" + token + "\" }";
+    } else {
+      body =
+          "<delegation-token><token>" + token + "</token></delegation-token>";
+    }
+    return body;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm

@@ -2707,3 +2707,223 @@ Server: Jetty(6.1.26)
 +---+
 
 
+* Cluster {Delegation Tokens API}
+
+  The Delegation Tokens API can be used to create, renew and cancel YARN ResourceManager delegation tokens. All delegation token requests must be carried out on a Kerberos authenticated connection(using SPNEGO). Carrying out operations on a non-kerberos connection will result in a FORBIDDEN response. In case of renewing a token, only the renewer specified when creating the token can renew the token. Other users(including the owner) are forbidden from renewing tokens. It should be noted that when cancelling or renewing a token, the token to be cancelled or renewed is specified by setting a header.
+
+  This feature is currently in the alpha stage and may change in the future.
+
+** URI
+
+  Use the following URI to create and cancel delegation tokens.
+
+------
+  * http://<rm http address:port>/ws/v1/cluster/delegation-token
+------
+
+  Use the following URI to renew delegation tokens.
+
+------
+  * http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
+------
+
+** HTTP Operations Supported
+
+------
+  * POST
+  * DELETE
+------
+
+** Query Parameters Supported
+
+------
+  None
+------
+
+** Elements of the <delegation-token> object
+
+  The response from the delegation tokens API contains one of the fields listed below.
+
+*---------------+--------------+-------------------------------+
+|| Item         || Data Type   || Description                   |
+*---------------+--------------+-------------------------------+
+| token         | string       | The delegation token          |
+*---------------+--------------+-------------------------------+
+| renewer       | string       | The user who is allowed to renew the delegation token |
+*---------------+--------------+-------------------------------+
+| owner         | string       | The owner of the delegation token |
+*---------------+--------------+-------------------------------+
+| kind          | string       | The kind of delegation token  |
+*---------------+--------------+-------------------------------+
+| expiration-time | long       | The expiration time of the token |
+*---------------+--------------+-------------------------------+
+| max-validity  | long         | The maximum validity of the token |
+*---------------+--------------+-------------------------------+
+
+** Response Examples
+
+*** Creating a token
+
+  <<JSON response>>
+
+  HTTP Request:
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token
+  Accept: application/json
+  Content-Type: application/json
+  {
+    "renewer" : "test-renewer"
+  }
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/json
++---+
+
+  Response body
+
++---+
+  {
+    "token":"MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckiEZpigFHSJTKaQECFN9EMM9BzfPoDxu572EVUpzqhnSGE1JNX0RFTEVHQVRJT05fVE9LRU4A",
+    "renewer":"test-renewer",
+    "owner":"client@EXAMPLE.COM",
+    "kind":"RM_DELEGATION_TOKEN",
+    "expiration-time":"1405153616489",
+    "max-validity":"1405672016489"
+  }
++---+
+
+  <<XML response>>
+
+  HTTP Request
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token
+  Accept: application/xml
+  Content-Type: application/xml
+  <delegation-token>
+    <renewer>test-renewer</renewer>
+  </delegation-token>
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Content-Length: 423
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/xml
++---+
+
+  Response Body
+
++---+
+  <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+  <delegation-token>
+    <token>MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUckgZ8yigFHSI4jMgcCFDTG8X6XFFn2udQngzSXQL8vWaKIE1JNX0RFTEVHQVRJT05fVE9LRU4A</token>
+    <renewer>test-renewer</renewer>
+    <owner>client@EXAMPLE.COM</owner>
+    <kind>RM_DELEGATION_TOKEN</kind>
+    <expiration-time>1405153180466</expiration-time>
+    <max-validity>1405671580466</max-validity>
+  </delegation-token>
++---+
+
+*** Renewing a token
+
+  <<JSON response>>
+
+  HTTP Request:
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
+  Accept: application/json
+  Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
+  Content-Type: application/json
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/json
++---+
+
+  Response body
+
++---+
+  {
+    "expiration-time":"1404112520402"
+  }
++---+
+
+  <<XML response>>
+
+  HTTP Request
+
+------
+  POST http://<rm http address:port>/ws/v1/cluster/delegation-token/expiration
+  Accept: application/xml
+  Content-Type: application/xml
+  Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
+------
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sat, 28 Jun 2014 18:08:11 GMT
+  Content-Length: 423
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/xml
++---+
+
+  Response Body
+
++---+
+  <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+  <delegation-token>
+    <expiration-time>1404112520402</expiration-time>
+  </delegation-token>
++---+
+
+*** Cancelling a token
+
+  HTTP Request
+
+-----
+DELETE http://<rm http address:port>/ws/v1/cluster/delegation-token
+Hadoop-YARN-RM-Delegation-Token: MgASY2xpZW50QEVYQU1QTEUuQ09NDHRlc3QtcmVuZXdlcgCKAUbjqcHHigFHB7ZFxwQCFKWD3znCkDSy6SQIjRCLDydxbxvgE1JNX0RFTEVHQVRJT05fVE9LRU4A
+Accept: application/xml
+-----
+
+  Response Header
+
++---+
+  HTTP/1.1 200 OK
+  WWW-Authenticate: Negotiate ...
+  Date: Sun, 29 Jun 2014 07:25:18 GMT
+  Transfer-Encoding: chunked
+  Server: Jetty(6.1.26)
+  Set-Cookie: ...
+  Content-Type: application/xml
++---+
+
+  No response body.