YARN-707. Added user information also in the YARN ClientToken so that AMs can implement authorization based on incoming users. Contributed by Jason Lowe.

svn merge --ignore-ancestry -c 1518868 ../../trunk/


git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1518869 13f79535-47bb-0310-9956-ffa450edef68

hadoop-yarn-project/CHANGES.txt

@@ -24,6 +24,9 @@ Release 2.1.1-beta - UNRELEASED
 
   INCOMPATIBLE CHANGES
 
+    YARN-707. Added user information also in the YARN ClientToken so that AMs
+    can implement authorization based on incoming users. (Jason Lowe via vinodkv)
+
   NEW FEATURES
 
   IMPROVEMENTS

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenIdentifier.java

@@ -39,6 +39,7 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
   public static final Text KIND_NAME = new Text("YARN_CLIENT_TOKEN");
 
   private ApplicationAttemptId applicationAttemptId;
+  private Text clientName = new Text();
 
   // TODO: Add more information in the tokenID such that it is not
   // transferrable, more secure etc.
@@ -46,21 +47,27 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
   public ClientToAMTokenIdentifier() {
   }
 
-  public ClientToAMTokenIdentifier(ApplicationAttemptId id) {
+  public ClientToAMTokenIdentifier(ApplicationAttemptId id, String client) {
     this();
     this.applicationAttemptId = id;
+    this.clientName = new Text(client);
   }
 
   public ApplicationAttemptId getApplicationAttemptID() {
     return this.applicationAttemptId;
   }
 
+  public String getClientName() {
+    return this.clientName.toString();
+  }
+
   @Override
   public void write(DataOutput out) throws IOException {
     out.writeLong(this.applicationAttemptId.getApplicationId()
       .getClusterTimestamp());
     out.writeInt(this.applicationAttemptId.getApplicationId().getId());
     out.writeInt(this.applicationAttemptId.getAttemptId());
+    this.clientName.write(out);
   }
 
   @Override
@@ -68,6 +75,7 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
     this.applicationAttemptId =
         ApplicationAttemptId.newInstance(
           ApplicationId.newInstance(in.readLong(), in.readInt()), in.readInt());
+    this.clientName.readFields(in);
   }
 
   @Override
@@ -77,10 +85,10 @@ public class ClientToAMTokenIdentifier extends TokenIdentifier {
 
   @Override
   public UserGroupInformation getUser() {
-    if (this.applicationAttemptId == null) {
+    if (this.clientName == null) {
       return null;
     }
-    return UserGroupInformation.createRemoteUser(this.applicationAttemptId.toString());
+    return UserGroupInformation.createRemoteUser(this.clientName.toString());
   }
 
   @InterfaceAudience.Private

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java

@@ -37,7 +37,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 public class ClientToAMTokenSecretManager extends
     BaseClientToAMTokenSecretManager {
 
-  // Only one client-token and one master-key for AM
+  // Only one master-key for AM
   private SecretKey masterKey;
 
   public ClientToAMTokenSecretManager(
@@ -53,7 +53,7 @@ public class ClientToAMTokenSecretManager extends
 
   @Override
   public SecretKey getMasterKey(ApplicationAttemptId applicationAttemptID) {
-    // Only one client-token and one master-key for AM, just return that.
+    // Only one master-key for AM, just return that.
     return this.masterKey;
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -248,7 +248,8 @@ public class ClientRMService extends AbstractService implements
     boolean allowAccess = checkAccess(callerUGI, application.getUser(),
         ApplicationAccessType.VIEW_APP, applicationId);
     ApplicationReport report =
-        application.createAndGetApplicationReport(allowAccess);
+        application.createAndGetApplicationReport(callerUGI.getUserName(),
+            allowAccess);
 
     GetApplicationReportResponse response = recordFactory
         .newRecordInstance(GetApplicationReportResponse.class);
@@ -425,7 +426,8 @@ public class ClientRMService extends AbstractService implements
       }
       boolean allowAccess = checkAccess(callerUGI, application.getUser(),
           ApplicationAccessType.VIEW_APP, application.getApplicationId());
-      reports.add(application.createAndGetApplicationReport(allowAccess));
+      reports.add(application.createAndGetApplicationReport(
+          callerUGI.getUserName(), allowAccess));
     }
 
     GetApplicationsResponse response =
@@ -471,7 +473,7 @@ public class ClientRMService extends AbstractService implements
             apps.size());
         for (RMApp app : apps) {
           if (app.getQueue().equals(queueInfo.getQueueName())) {
-            appReports.add(app.createAndGetApplicationReport(true));
+            appReports.add(app.createAndGetApplicationReport(null, true));
           }
         }
       }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java

@@ -24,6 +24,8 @@ import java.util.HashSet;
 import java.util.Map;
 import java.util.Set;
 
+import javax.crypto.SecretKey;
+
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -44,7 +46,6 @@ import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
@@ -75,14 +76,14 @@ public abstract class RMStateStore extends AbstractService {
   public static class ApplicationAttemptState {
     final ApplicationAttemptId attemptId;
     final Container masterContainer;
-    final Credentials appAttemptTokens;
+    final Credentials appAttemptCredentials;
 
     public ApplicationAttemptState(ApplicationAttemptId attemptId,
         Container masterContainer,
-        Credentials appAttemptTokens) {
+        Credentials appAttemptCredentials) {
       this.attemptId = attemptId;
       this.masterContainer = masterContainer;
-      this.appAttemptTokens = appAttemptTokens;
+      this.appAttemptCredentials = appAttemptCredentials;
     }
 
     public Container getMasterContainer() {
@@ -91,8 +92,8 @@ public abstract class RMStateStore extends AbstractService {
     public ApplicationAttemptId getAttemptId() {
       return attemptId;
     }
-    public Credentials getAppAttemptTokens() {
+    public Credentials getAppAttemptCredentials() {
-      return appAttemptTokens;
+      return appAttemptCredentials;
     }
   }
   
@@ -265,7 +266,7 @@ public abstract class RMStateStore extends AbstractService {
    * RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt
    */
   public synchronized void storeApplicationAttempt(RMAppAttempt appAttempt) {
-    Credentials credentials = getTokensFromAppAttempt(appAttempt);
+    Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
 
     ApplicationAttemptState attemptState =
         new ApplicationAttemptState(appAttempt.getAppAttemptId(),
@@ -365,7 +366,7 @@ public abstract class RMStateStore extends AbstractService {
             app.getSubmitTime(), app.getApplicationSubmissionContext(),
             app.getUser());
     for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
-      Credentials credentials = getTokensFromAppAttempt(appAttempt);
+      Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
       ApplicationAttemptState attemptState =
           new ApplicationAttemptState(appAttempt.getAppAttemptId(),
             appAttempt.getMasterContainer(), credentials);
@@ -395,17 +396,21 @@ public abstract class RMStateStore extends AbstractService {
   // YARN-986 
   public static final Text AM_RM_TOKEN_SERVICE = new Text(
     "AM_RM_TOKEN_SERVICE");
+
+  public static final Text AM_CLIENT_TOKEN_MASTER_KEY_NAME =
+      new Text("YARN_CLIENT_TOKEN_MASTER_KEY");
   
-  private Credentials getTokensFromAppAttempt(RMAppAttempt appAttempt) {
+  private Credentials getCredentialsFromAppAttempt(RMAppAttempt appAttempt) {
     Credentials credentials = new Credentials();
     Token<AMRMTokenIdentifier> appToken = appAttempt.getAMRMToken();
     if(appToken != null){
       credentials.addToken(AM_RM_TOKEN_SERVICE, appToken);
     }
-    Token<ClientToAMTokenIdentifier> clientToAMToken =
+    SecretKey clientTokenMasterKey =
-        appAttempt.getClientToAMToken();
+        appAttempt.getClientTokenMasterKey();
-    if(clientToAMToken != null){
+    if(clientTokenMasterKey != null){
-      credentials.addToken(clientToAMToken.getService(), clientToAMToken);
+      credentials.addSecretKey(AM_CLIENT_TOKEN_MASTER_KEY_NAME,
+          clientTokenMasterKey.getEncoded());
     }
     return credentials;
   }
@@ -445,7 +450,7 @@ public abstract class RMStateStore extends AbstractService {
                     ((RMStateStoreAppAttemptEvent) event).getAppAttemptState();
           Exception storedException = null;
 
-          Credentials credentials = attemptState.getAppAttemptTokens();
+          Credentials credentials = attemptState.getAppAttemptCredentials();
           ByteBuffer appAttemptTokens = null;
           try {
             if(credentials != null){

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java

@@ -128,10 +128,12 @@ public interface RMApp extends EventHandler<RMAppEvent> {
    *   <li>resource usage report - all values are -1</li>
    * </ul>
    *
+   * @param clientUserName the user name of the client requesting the report
    * @param allowAccess whether to allow full access to the report
    * @return the {@link ApplicationReport} detailing the status of the application.
    */
-  ApplicationReport createAndGetApplicationReport(boolean allowAccess);
+  ApplicationReport createAndGetApplicationReport(String clientUserName,
+      boolean allowAccess);
   
   /**
    * To receive the collection of all {@link RMNode}s whose updates have been

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java

@@ -18,7 +18,6 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
 
-import java.io.IOException;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.EnumSet;
@@ -411,7 +410,8 @@ public class RMAppImpl implements RMApp, Recoverable {
   }
   
   @Override
-  public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+  public ApplicationReport createAndGetApplicationReport(String clientUserName,
+      boolean allowAccess) {
     this.readLock.lock();
 
     try {
@@ -432,15 +432,18 @@ public class RMAppImpl implements RMApp, Recoverable {
           currentApplicationAttemptId = this.currentAttempt.getAppAttemptId();
           trackingUrl = this.currentAttempt.getTrackingUrl();
           origTrackingUrl = this.currentAttempt.getOriginalTrackingUrl();
-          Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
+          if (UserGroupInformation.isSecurityEnabled()
-              this.currentAttempt.getClientToAMToken();
+              && clientUserName != null) {
-          if (attemptClientToAMToken != null) {
+            Token<ClientToAMTokenIdentifier> attemptClientToAMToken =
-            clientToAMToken =
+                new Token<ClientToAMTokenIdentifier>(
-                BuilderUtils.newClientToAMToken(
+                    new ClientToAMTokenIdentifier(
-                    attemptClientToAMToken.getIdentifier(),
+                        currentApplicationAttemptId, clientUserName),
-                    attemptClientToAMToken.getKind().toString(),
+                        rmContext.getClientToAMTokenSecretManager());
-                    attemptClientToAMToken.getPassword(),
+            clientToAMToken = BuilderUtils.newClientToAMToken(
-                    attemptClientToAMToken.getService().toString());
+                attemptClientToAMToken.getIdentifier(),
+                attemptClientToAMToken.getKind().toString(),
+                attemptClientToAMToken.getPassword(),
+                attemptClientToAMToken.getService().toString());
           }
           host = this.currentAttempt.getHost();
           rpcPort = this.currentAttempt.getRpcPort();
@@ -451,20 +454,15 @@ public class RMAppImpl implements RMApp, Recoverable {
 
         if (currentAttempt != null && 
             currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
-          try {
+          if (getApplicationSubmissionContext().getUnmanagedAM() &&
-            if (getApplicationSubmissionContext().getUnmanagedAM() &&
+              clientUserName != null && getUser().equals(clientUserName)) {
-              getUser().equals(UserGroupInformation.getCurrentUser().getUserName())) {
+            Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
-              Token<AMRMTokenIdentifier> token = currentAttempt.getAMRMToken();
+            if (token != null) {
-              if (token != null) {
+              amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(),
-                amrmToken = BuilderUtils.newAMRMToken(token.getIdentifier(),
+                  token.getKind().toString(), token.getPassword(),
-                    token.getKind().toString(), token.getPassword(),
+                  token.getService().toString());
-                    token.getService().toString());
-              }
             }
-          } catch (IOException ex) {
+          }
-            LOG.warn("UserGroupInformation.getCurrentUser() error: " + 
-              ex.toString(), ex);
-          }          
         }
       }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java

@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
 import java.util.List;
 import java.util.Set;
 
+import javax.crypto.SecretKey;
+
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -32,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.NodeId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 
 /**
@@ -90,12 +91,6 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
    */
   String getWebProxyBase();
 
-  /**
-   * The token required by the clients to talk to the application attempt
-   * @return the token required by the clients to talk to the application attempt
-   */
-  Token<ClientToAMTokenIdentifier> getClientToAMToken();
-
   /**
    * Diagnostics information for the application attempt.
    * @return diagnostics information for the application attempt.
@@ -154,6 +149,12 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
    */
   Token<AMRMTokenIdentifier> getAMRMToken();
 
+  /**
+   * The master key for client-to-AM tokens for this app attempt
+   * @return The master key for client-to-AM tokens for this app attempt
+   */
+  SecretKey getClientTokenMasterKey();
+
   /**
    * Get application container and resource usage information.
    * @return an ApplicationResourceUsageReport object.

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java

@@ -33,12 +33,13 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
 import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
 
+import javax.crypto.SecretKey;
+
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.io.Text;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
@@ -60,8 +61,6 @@ import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenSelector;
 import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEvent;
@@ -126,9 +125,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
   private final WriteLock writeLock;
 
   private final ApplicationAttemptId applicationAttemptId;
-  private Token<ClientToAMTokenIdentifier> clientToAMToken;
   private final ApplicationSubmissionContext submissionContext;
   private Token<AMRMTokenIdentifier> amrmToken = null;
+  private SecretKey clientTokenMasterKey = null;
 
   //nodes on while this attempt's containers ran
   private final Set<NodeId> ranNodes =
@@ -499,8 +498,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
   }
 
   @Override
-  public Token<ClientToAMTokenIdentifier> getClientToAMToken() {
+  public SecretKey getClientTokenMasterKey() {
-    return this.clientToAMToken;
+    return this.clientTokenMasterKey;
   }
 
   @Override
@@ -659,7 +658,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     ApplicationAttemptState attemptState = appState.getAttempt(getAppAttemptId());
     assert attemptState != null;
     setMasterContainer(attemptState.getMasterContainer());
-    recoverAppAttemptTokens(attemptState.getAppAttemptTokens());
+    recoverAppAttemptCredentials(attemptState.getAppAttemptCredentials());
     LOG.info("Recovered attempt: AppId: " + getAppAttemptId().getApplicationId()
              + " AttemptId: " + getAppAttemptId()
              + " MasterContainer: " + masterContainer);
@@ -668,17 +667,16 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
                                  RMAppAttemptEventType.RECOVER));
   }
 
-  private void recoverAppAttemptTokens(Credentials appAttemptTokens) {
+  private void recoverAppAttemptCredentials(Credentials appAttemptTokens) {
     if (appAttemptTokens == null) {
       return;
     }
-    if (UserGroupInformation.isSecurityEnabled()) {
 
-      ClientToAMTokenSelector clientToAMTokenSelector =
+    if (UserGroupInformation.isSecurityEnabled()) {
-          new ClientToAMTokenSelector();
+      byte[] clientTokenMasterKeyBytes = appAttemptTokens.getSecretKey(
-      this.clientToAMToken =
+          RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME);
-          clientToAMTokenSelector.selectToken(new Text(),
+      clientTokenMasterKey = rmContext.getClientToAMTokenSecretManager()
-            appAttemptTokens.getAllTokens());
+          .registerMasterKey(applicationAttemptId, clientTokenMasterKeyBytes);
     }
 
     // Only one AMRMToken is stored per-attempt, so this should be fine. Can't
@@ -715,15 +713,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
           .registerAppAttempt(appAttempt.applicationAttemptId);
 
       if (UserGroupInformation.isSecurityEnabled()) {
-
+        appAttempt.clientTokenMasterKey = appAttempt.rmContext
-        appAttempt.rmContext.getClientToAMTokenSecretManager()
+            .getClientToAMTokenSecretManager()
-          .registerApplication(appAttempt.applicationAttemptId);
+            .registerApplication(appAttempt.applicationAttemptId);
-
-        // create clientToAMToken
-        appAttempt.clientToAMToken =
-            new Token<ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifier(
-              appAttempt.applicationAttemptId),
-              appAttempt.rmContext.getClientToAMTokenSecretManager());
       }
 
       // create AMRMToken
@@ -762,7 +754,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
               message)
           );
 
-      appAttempt.removeTokens(appAttempt);
+      appAttempt.removeCredentials(appAttempt);
     }
   }
 
@@ -895,7 +887,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
       appAttempt.eventHandler.handle(new AppRemovedSchedulerEvent(appAttemptId,
         finalAttemptState));
 
-      appAttempt.removeTokens(appAttempt);
+      appAttempt.removeCredentials(appAttempt);
     }
   }
 
@@ -1256,7 +1248,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
     store.storeApplicationAttempt(this);
   }
 
-  private void removeTokens(RMAppAttemptImpl appAttempt) {
+  private void removeCredentials(RMAppAttemptImpl appAttempt) {
     // Unregister from the ClientToAMTokenSecretManager
     if (UserGroupInformation.isSecurityEnabled()) {
       appAttempt.rmContext.getClientToAMTokenSecretManager()

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java

@@ -33,9 +33,18 @@ public class ClientToAMTokenSecretManagerInRM extends
   private Map<ApplicationAttemptId, SecretKey> masterKeys =
       new HashMap<ApplicationAttemptId, SecretKey>();
 
-  public synchronized void registerApplication(
+  public synchronized SecretKey registerApplication(
       ApplicationAttemptId applicationAttemptID) {
-    this.masterKeys.put(applicationAttemptID, generateSecret());
+    SecretKey key = generateSecret();
+    this.masterKeys.put(applicationAttemptID, key);
+    return key;
+  }
+
+  public synchronized SecretKey registerMasterKey(
+      ApplicationAttemptId applicationAttemptID, byte[] keyData) {
+    SecretKey key = createSecretKey(keyData);
+    this.masterKeys.put(applicationAttemptID, key);
+    return key;
   }
 
   public synchronized void unRegisterApplication(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java

@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
 import org.apache.hadoop.yarn.util.Records;
 import org.apache.log4j.Level;
@@ -387,6 +388,10 @@ public class MockRM extends ResourceManager {
     return this.rmDTSecretManager;
   }
 
+  public ClientToAMTokenSecretManagerInRM getClientToAMTokenSecretManager() {
+    return this.clientToAMSecretManager;
+  }
+
   @Override
   protected void startWepApp() {
     // override to disable webapp

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java

@@ -541,16 +541,21 @@ public class TestRMRestart {
     Assert.assertEquals(BuilderUtils.newContainerId(attemptId1, 1),
       attemptState.getMasterContainer().getId());
 
-    // the appToken and clientToAMToken that are generated when RMAppAttempt
+    // the appToken and clientTokenMasterKey that are generated when
-    // is created,
+    // RMAppAttempt is created,
     HashSet<Token<?>> tokenSet = new HashSet<Token<?>>();
     tokenSet.add(attempt1.getAMRMToken());
-    tokenSet.add(attempt1.getClientToAMToken());
+    byte[] clientTokenMasterKey =
+        attempt1.getClientTokenMasterKey().getEncoded();
 
-    // assert application Token is saved
+    // assert application credentials are saved
+    Credentials savedCredentials = attemptState.getAppAttemptCredentials();
     HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
-    savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+    savedTokens.addAll(savedCredentials.getAllTokens());
     Assert.assertEquals(tokenSet, savedTokens);
+    Assert.assertArrayEquals("client token master key not saved",
+        clientTokenMasterKey, savedCredentials.getSecretKey(
+            RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
 
     // start new RM
     MockRM rm2 = new TestSecurityMockRM(conf, memStore);
@@ -564,13 +569,18 @@ public class TestRMRestart {
     Assert.assertNotNull(loadedAttempt1);
     savedTokens.clear();
     savedTokens.add(loadedAttempt1.getAMRMToken());
-    savedTokens.add(loadedAttempt1.getClientToAMToken());
     Assert.assertEquals(tokenSet, savedTokens);
 
-    // assert clientToAMToken is recovered back to api-versioned
+    // assert client token master key is recovered back to api-versioned
-    // clientToAMToken
+    // client token master key
-    Assert.assertEquals(attempt1.getClientToAMToken(),
+    Assert.assertEquals("client token master key not restored",
-      loadedAttempt1.getClientToAMToken());
+        attempt1.getClientTokenMasterKey(),
+        loadedAttempt1.getClientTokenMasterKey());
+
+    // assert secret manager also knows about the key
+    Assert.assertArrayEquals(clientTokenMasterKey,
+        rm2.getClientToAMTokenSecretManager().getMasterKey(attemptId1)
+            .getEncoded());
 
     // Not testing ApplicationTokenSecretManager has the password populated back,
     // that is needed in work-preserving restart

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java

@@ -115,7 +115,8 @@ public abstract class MockAsm extends MockApps {
       throw new UnsupportedOperationException("Not supported yet.");
     }
     @Override
-    public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+    public ApplicationReport createAndGetApplicationReport(
+        String clientUserName,boolean allowAccess) {
       throw new UnsupportedOperationException("Not supported yet.");
     }
     @Override

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java

@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.resourcemanager.recovery;
 
+import static org.junit.Assert.assertArrayEquals;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertTrue;
@@ -25,12 +26,12 @@ import static org.junit.Assert.fail;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.List;
 import java.util.Map;
 
+import javax.crypto.SecretKey;
+
 import junit.framework.Assert;
 
 import org.apache.commons.logging.Log;
@@ -55,7 +56,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationAttemptState;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
@@ -198,7 +198,7 @@ public class TestRMStateStore {
 
   ContainerId storeAttempt(RMStateStore store, ApplicationAttemptId attemptId,
       String containerIdStr, Token<AMRMTokenIdentifier> appToken,
-      Token<ClientToAMTokenIdentifier> clientToAMToken, TestDispatcher dispatcher)
+      SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
       throws Exception {
 
     Container container = new ContainerPBImpl();
@@ -207,7 +207,8 @@ public class TestRMStateStore {
     when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
     when(mockAttempt.getMasterContainer()).thenReturn(container);
     when(mockAttempt.getAMRMToken()).thenReturn(appToken);
-    when(mockAttempt.getClientToAMToken()).thenReturn(clientToAMToken);
+    when(mockAttempt.getClientTokenMasterKey())
+        .thenReturn(clientTokenMasterKey);
     dispatcher.attemptId = attemptId;
     dispatcher.storedException = null;
     store.storeApplicationAttempt(mockAttempt);
@@ -215,7 +216,6 @@ public class TestRMStateStore {
     return container.getId();
   }
 
-  @SuppressWarnings("unchecked")
   void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws Exception {
     long submitTime = System.currentTimeMillis();
     Configuration conf = new YarnConfiguration();
@@ -233,33 +233,33 @@ public class TestRMStateStore {
     ApplicationId appId1 = attemptId1.getApplicationId();
     storeApp(store, appId1, submitTime);
 
-    // create application token1 for attempt1
+    // create application token and client token key for attempt1
-    List<Token<?>> appAttemptToken1 =
+    Token<AMRMTokenIdentifier> appAttemptToken1 =
-        generateTokens(attemptId1, appTokenMgr, clientToAMTokenMgr, conf);
+        generateAMRMToken(attemptId1, appTokenMgr);
     HashSet<Token<?>> attemptTokenSet1 = new HashSet<Token<?>>();
-    attemptTokenSet1.addAll(appAttemptToken1);
+    attemptTokenSet1.add(appAttemptToken1);
+    SecretKey clientTokenKey1 =
+        clientToAMTokenMgr.registerApplication(attemptId1);
 
     ContainerId containerId1 = storeAttempt(store, attemptId1,
           "container_1352994193343_0001_01_000001",
-          (Token<AMRMTokenIdentifier>) (appAttemptToken1.get(0)),
+          appAttemptToken1, clientTokenKey1, dispatcher);
-          (Token<ClientToAMTokenIdentifier>)(appAttemptToken1.get(1)),
-          dispatcher);
 
     String appAttemptIdStr2 = "appattempt_1352994193343_0001_000002";
     ApplicationAttemptId attemptId2 =
         ConverterUtils.toApplicationAttemptId(appAttemptIdStr2);
 
-    // create application token2 for attempt2
+    // create application token and client token key for attempt2
-    List<Token<?>> appAttemptToken2 =
+    Token<AMRMTokenIdentifier> appAttemptToken2 =
-        generateTokens(attemptId2, appTokenMgr, clientToAMTokenMgr, conf);
+        generateAMRMToken(attemptId2, appTokenMgr);
     HashSet<Token<?>> attemptTokenSet2 = new HashSet<Token<?>>();
-    attemptTokenSet2.addAll(appAttemptToken2);
+    attemptTokenSet2.add(appAttemptToken2);
+    SecretKey clientTokenKey2 =
+        clientToAMTokenMgr.registerApplication(attemptId2);
 
     ContainerId containerId2 = storeAttempt(store, attemptId2,
           "container_1352994193343_0001_02_000001",
-          (Token<AMRMTokenIdentifier>) (appAttemptToken2.get(0)),
+          appAttemptToken2, clientTokenKey2, dispatcher);
-          (Token<ClientToAMTokenIdentifier>)(appAttemptToken2.get(1)),
-          dispatcher);
 
     ApplicationAttemptId attemptIdRemoved = ConverterUtils
         .toApplicationAttemptId("appattempt_1352994193343_0002_000001");
@@ -306,8 +306,12 @@ public class TestRMStateStore {
     assertEquals(containerId1, attemptState.getMasterContainer().getId());
     // attempt1 applicationToken is loaded correctly
     HashSet<Token<?>> savedTokens = new HashSet<Token<?>>();
-    savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+    savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
     assertEquals(attemptTokenSet1, savedTokens);
+    // attempt1 client token master key is loaded correctly
+    assertArrayEquals(clientTokenKey1.getEncoded(),
+        attemptState.getAppAttemptCredentials()
+        .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
 
     attemptState = appState.getAttempt(attemptId2);
     // attempt2 is loaded correctly
@@ -317,8 +321,12 @@ public class TestRMStateStore {
     assertEquals(containerId2, attemptState.getMasterContainer().getId());
     // attempt2 applicationToken is loaded correctly
     savedTokens.clear();
-    savedTokens.addAll(attemptState.getAppAttemptTokens().getAllTokens());
+    savedTokens.addAll(attemptState.getAppAttemptCredentials().getAllTokens());
     assertEquals(attemptTokenSet2, savedTokens);
+    // attempt2 client token master key is loaded correctly
+    assertArrayEquals(clientTokenKey2.getEncoded(),
+        attemptState.getAppAttemptCredentials()
+        .getSecretKey(RMStateStore.AM_CLIENT_TOKEN_MASTER_KEY_NAME));
 
     // assert store is in expected state after everything is cleaned
     assertTrue(stateStoreHelper.isFinalStateValid());
@@ -357,24 +365,14 @@ public class TestRMStateStore {
     Assert.assertEquals(sequenceNumber, secretManagerState.getDTSequenceNumber());
   }
 
-  private List<Token<?>> generateTokens(ApplicationAttemptId attemptId,
+  private Token<AMRMTokenIdentifier> generateAMRMToken(
-      AMRMTokenSecretManager appTokenMgr,
+      ApplicationAttemptId attemptId,
-      ClientToAMTokenSecretManagerInRM clientToAMTokenMgr, Configuration conf) {
+      AMRMTokenSecretManager appTokenMgr) {
     AMRMTokenIdentifier appTokenId =
         new AMRMTokenIdentifier(attemptId);
     Token<AMRMTokenIdentifier> appToken =
         new Token<AMRMTokenIdentifier>(appTokenId, appTokenMgr);
     appToken.setService(new Text("appToken service"));
-
+    return appToken;
-    ClientToAMTokenIdentifier clientToAMTokenId =
-        new ClientToAMTokenIdentifier(attemptId);
-    clientToAMTokenMgr.registerApplication(attemptId);
-    Token<ClientToAMTokenIdentifier> clientToAMToken =
-        new Token<ClientToAMTokenIdentifier>(clientToAMTokenId, clientToAMTokenMgr);
-    clientToAMToken.setService(new Text("clientToAMToken service"));
-    List<Token<?>> tokenPair = new ArrayList<Token<?>>();
-    tokenPair.add(0, appToken);
-    tokenPair.add(1, clientToAMToken);
-    return tokenPair;
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java

@@ -143,7 +143,8 @@ public class MockRMApp implements RMApp {
   }
 
   @Override
-  public ApplicationReport createAndGetApplicationReport(boolean allowAccess) {
+  public ApplicationReport createAndGetApplicationReport(
+      String clientUserName, boolean allowAccess) {
     throw new UnsupportedOperationException("Not supported yet.");
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java

@@ -726,7 +726,9 @@ public class TestRMAppTransitions {
   public void testGetAppReport() {
     RMApp app = createNewTestApp(null);
     assertAppState(RMAppState.NEW, app);
-    ApplicationReport report = app.createAndGetApplicationReport(true);
+    ApplicationReport report = app.createAndGetApplicationReport(null, true);
+    Assert.assertNotNull(report.getApplicationResourceUsageReport());
+    report = app.createAndGetApplicationReport("clientuser", true);
     Assert.assertNotNull(report.getApplicationResourceUsageReport());
   }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java

@@ -115,7 +115,6 @@ public class TestClientToAMTokens {
     private final byte[] secretKey;
     private InetSocketAddress address;
     private boolean pinged = false;
-    private ClientToAMTokenSecretManager secretManager;
     
     public CustomAM(ApplicationAttemptId appId, byte[] secretKey) {
       super("CustomAM");
@@ -132,12 +131,14 @@ public class TestClientToAMTokens {
     protected void serviceStart() throws Exception {
       Configuration conf = getConfig();
 
-      secretManager = new ClientToAMTokenSecretManager(this.appAttemptId, secretKey);
       Server server;
       try {
         server =
-            new RPC.Builder(conf).setProtocol(CustomProtocol.class)
+            new RPC.Builder(conf)
-              .setNumHandlers(1).setSecretManager(secretManager)
+              .setProtocol(CustomProtocol.class)
+              .setNumHandlers(1)
+              .setSecretManager(
+                new ClientToAMTokenSecretManager(this.appAttemptId, secretKey))
               .setInstance(this).build();
       } catch (Exception e) {
         throw new YarnRuntimeException(e);
@@ -146,14 +147,10 @@ public class TestClientToAMTokens {
       this.address = NetUtils.getConnectAddress(server);
       super.serviceStart();
     }
-    
-    public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
-      return this.secretManager;
-    }
   }
 
   @Test
-  public void testClientToAMs() throws Exception {
+  public void testClientToAMTokens() throws Exception {
 
     final Configuration conf = new Configuration();
     conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
@@ -204,7 +201,7 @@ public class TestClientToAMTokens {
     GetApplicationReportResponse reportResponse =
         rm.getClientRMService().getApplicationReport(request);
     ApplicationReport appReport = reportResponse.getApplicationReport();
-    org.apache.hadoop.yarn.api.records.Token clientToAMToken =
+    org.apache.hadoop.yarn.api.records.Token originalClientToAMToken =
         appReport.getClientToAMToken();
 
     ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
@@ -259,17 +256,47 @@ public class TestClientToAMTokens {
       Assert.assertFalse(am.pinged);
     }
 
-    // Verify denial for a malicious user
-    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
     Token<ClientToAMTokenIdentifier> token =
-        ConverterUtils.convertFromYarn(clientToAMToken, am.address);
+        ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
 
+    // Verify denial for a malicious user with tampered ID
+    verifyTokenWithTamperedID(conf, am, token);
+
+    // Verify denial for a malicious user with tampered user-name
+    verifyTokenWithTamperedUserName(conf, am, token);
+
+    // Now for an authenticated user
+    verifyValidToken(conf, am, token);
+  }
+
+  private void verifyTokenWithTamperedID(final Configuration conf,
+      final CustomAM am, Token<ClientToAMTokenIdentifier> token)
+      throws IOException {
     // Malicious user, messes with appId
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
     ClientToAMTokenIdentifier maliciousID =
         new ClientToAMTokenIdentifier(BuilderUtils.newApplicationAttemptId(
-          BuilderUtils.newApplicationId(app.getApplicationId()
+          BuilderUtils.newApplicationId(am.appAttemptId.getApplicationId()
-            .getClusterTimestamp(), 42), 43));
+            .getClusterTimestamp(), 42), 43), UserGroupInformation
+          .getCurrentUser().getShortUserName());
 
+    verifyTamperedToken(conf, am, token, ugi, maliciousID);
+  }
+
+  private void verifyTokenWithTamperedUserName(final Configuration conf,
+      final CustomAM am, Token<ClientToAMTokenIdentifier> token)
+      throws IOException {
+    // Malicious user, messes with appId
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
+    ClientToAMTokenIdentifier maliciousID =
+        new ClientToAMTokenIdentifier(am.appAttemptId, "evilOrc");
+
+    verifyTamperedToken(conf, am, token, ugi, maliciousID);
+  }
+
+  private void verifyTamperedToken(final Configuration conf, final CustomAM am,
+      Token<ClientToAMTokenIdentifier> token, UserGroupInformation ugi,
+      ClientToAMTokenIdentifier maliciousID) {
     Token<ClientToAMTokenIdentifier> maliciousToken =
         new Token<ClientToAMTokenIdentifier>(maliciousID.getBytes(),
           token.getPassword(), token.getKind(),
@@ -309,8 +336,12 @@ public class TestClientToAMTokens {
               + "Mismatched response."));
       Assert.assertFalse(am.pinged);
     }
+  }
 
-    // Now for an authenticated user
+  private void verifyValidToken(final Configuration conf, final CustomAM am,
+      Token<ClientToAMTokenIdentifier> token) throws IOException,
+      InterruptedException {
+    UserGroupInformation ugi;
     ugi = UserGroupInformation.createRemoteUser("me");
     ugi.addToken(token);
 
@@ -326,5 +357,4 @@ public class TestClientToAMTokens {
       }
     });
   }
-
 }