merge YARN-134 from trunk. Fixes ClientToAMSecretManager creates keys without checking for validity of the appID. (Contributed by Vinod Kumar Vavilapalli)

git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-2@1395843 13f79535-47bb-0310-9956-ffa450edef68

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/client/MRClientService.java

@@ -82,8 +82,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
 import org.apache.hadoop.yarn.service.AbstractService;
 import org.apache.hadoop.yarn.webapp.WebApp;
 import org.apache.hadoop.yarn.webapp.WebApps;
@@ -115,16 +114,15 @@ public class MRClientService extends AbstractService
     YarnRPC rpc = YarnRPC.create(conf);
     InetSocketAddress address = new InetSocketAddress(0);
 
-    ClientToAMSecretManager secretManager = null;
+    ClientToAMTokenSecretManager secretManager = null;
     if (UserGroupInformation.isSecurityEnabled()) {
-      secretManager = new ClientToAMSecretManager();
       String secretKeyStr =
           System
               .getenv(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
       byte[] bytes = Base64.decodeBase64(secretKeyStr);
-      ClientTokenIdentifier identifier = new ClientTokenIdentifier(
+      secretManager =
-          this.appContext.getApplicationID());
+          new ClientToAMTokenSecretManager(this.appContext.getApplicationID(),
-      secretManager.setMasterKey(identifier, bytes);
+            bytes);
     }
     server =
         rpc.getServer(MRClientProtocol.class, protocolHandler, address,

hadoop-yarn-project/CHANGES.txt

@@ -45,6 +45,9 @@ Release 2.0.3-alpha - Unreleased
     YARN-102. Move the apache header to the top of the file in MemStore.java.
     (Devaraj K via sseth)
     
+    YARN-134. ClientToAMSecretManager creates keys without checking for
+    validity of the appID. (Vinod Kumar Vavilapalli via sseth)
+
 Release 2.0.2-alpha - 2012-09-07 
 
     YARN-9. Rename YARN_HOME to HADOOP_YARN_HOME. (vinodkv via acmurthy)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java

@@ -34,7 +34,7 @@ public interface ApplicationConstants {
 
   // TODO: They say tokens via env isn't good.
   public static final String APPLICATION_CLIENT_SECRET_ENV_NAME =
-    "AppClientTokenEnv";
+    "AppClientSecretEnv";
   
   /**
    * The environment variable for CONTAINER_ID. Set in AppMaster environment

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/BaseClientToAMTokenSecretManager.java

@@ -0,0 +1,53 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.security.client;
+
+import javax.crypto.SecretKey;
+
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+public abstract class BaseClientToAMTokenSecretManager extends
+    SecretManager<ClientTokenIdentifier> {
+
+  public abstract SecretKey getMasterKey(ApplicationId applicationId);
+
+  @Override
+  public synchronized byte[] createPassword(
+      ClientTokenIdentifier identifier) {
+    return createPassword(identifier.getBytes(),
+      getMasterKey(identifier.getApplicationID()));
+  }
+
+  @Override
+  public byte[] retrievePassword(ClientTokenIdentifier identifier)
+      throws SecretManager.InvalidToken {
+    SecretKey masterKey = getMasterKey(identifier.getApplicationID());
+    if (masterKey == null) {
+      throw new SecretManager.InvalidToken("Illegal client-token!");
+    }
+    return createPassword(identifier.getBytes(), masterKey);
+  }
+
+  @Override
+  public ClientTokenIdentifier createIdentifier() {
+    return new ClientTokenIdentifier();
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMSecretManager.java

@@ -1,103 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements.  See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership.  The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License.  You may obtain a copy of the License at
-*
-*     http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.yarn.security.client;
-
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.crypto.SecretKey;
-
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.security.token.SecretManager;
-
-public class ClientToAMSecretManager extends
-    SecretManager<ClientTokenIdentifier> {
-
-  private static Log LOG = LogFactory.getLog(ClientToAMSecretManager.class);
-
-  // Per application masterkeys for managing client-tokens
-  private Map<Text, SecretKey> masterKeys = new HashMap<Text, SecretKey>();
-
-  public void setMasterKey(ClientTokenIdentifier identifier, byte[] key) {
-    SecretKey sk = SecretManager.createSecretKey(key);
-    Text applicationID = identifier.getApplicationID();
-    this.masterKeys.put(applicationID, sk);
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Setting master key for "
-          + applicationID
-          + " as "
-          + new String(Base64.encodeBase64(this.masterKeys.get(applicationID)
-              .getEncoded())));
-    }
-  }
-
-  private void addMasterKey(ClientTokenIdentifier identifier) {
-    Text applicationID = identifier.getApplicationID();
-    this.masterKeys.put(applicationID, generateSecret());
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Creating master key for "
-          + applicationID
-          + " as "
-          + new String(Base64.encodeBase64(this.masterKeys.get(applicationID)
-              .getEncoded())));}
-  }
-
-  // TODO: Handle the masterKey invalidation.
-  public synchronized SecretKey getMasterKey(
-      ClientTokenIdentifier identifier) {
-    Text applicationID = identifier.getApplicationID();
-    if (!this.masterKeys.containsKey(applicationID)) {
-      addMasterKey(identifier);
-    }
-    return this.masterKeys.get(applicationID);
-  }
-
-  @Override
-  public synchronized byte[] createPassword(
-      ClientTokenIdentifier identifier) {
-    byte[] password =
-        createPassword(identifier.getBytes(), getMasterKey(identifier));
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Password created is "
-          + new String(Base64.encodeBase64(password)));
-    }
-    return password;
-  }
-
-  @Override
-  public byte[] retrievePassword(ClientTokenIdentifier identifier)
-      throws SecretManager.InvalidToken {
-    byte[] password =
-        createPassword(identifier.getBytes(), getMasterKey(identifier));
-    if (LOG.isDebugEnabled()) {
-      LOG.debug("Password retrieved is "
-          + new String(Base64.encodeBase64(password)));
-    }
-    return password;
-  }
-
-  @Override
-  public ClientTokenIdentifier createIdentifier() {
-    return new ClientTokenIdentifier();
-  }
-
-}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientToAMTokenSecretManager.java

@@ -0,0 +1,44 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.security.client;
+
+import javax.crypto.SecretKey;
+
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+public class ClientToAMTokenSecretManager extends
+    BaseClientToAMTokenSecretManager {
+
+  // Only one client-token and one master-key for AM
+  private final SecretKey masterKey;
+
+  public ClientToAMTokenSecretManager(ApplicationId applicationID,
+      byte[] secretKeyBytes) {
+    super();
+    this.masterKey = SecretManager.createSecretKey(secretKeyBytes);
+  }
+
+  @Override
+  public SecretKey getMasterKey(ApplicationId applicationID) {
+    // Only one client-token and one master-key for AM, just return that.
+    return this.masterKey;
+  }
+
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/client/ClientTokenIdentifier.java

@@ -28,36 +28,39 @@ import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenIdentifier;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.BuilderUtils;
 
 public class ClientTokenIdentifier extends TokenIdentifier {
 
   public static final Text KIND_NAME = new Text("YARN_CLIENT_TOKEN");
 
-  private Text appId;
+  private ApplicationId applicationId;
 
   // TODO: Add more information in the tokenID such that it is not
   // transferrable, more secure etc.
 
-  public ClientTokenIdentifier(ApplicationId id) {
-    this.appId = new Text(Integer.toString(id.getId()));
-  }
-
   public ClientTokenIdentifier() {
-    this.appId = new Text();
   }
 
-  public Text getApplicationID() {
+  public ClientTokenIdentifier(ApplicationId id) {
-    return appId;
+    this();
+    this.applicationId = id;
+  }
+
+  public ApplicationId getApplicationID() {
+    return this.applicationId;
   }
 
   @Override
   public void write(DataOutput out) throws IOException {
-    appId.write(out);
+    out.writeLong(this.applicationId.getClusterTimestamp());
+    out.writeInt(this.applicationId.getId());
   }
 
   @Override
   public void readFields(DataInput in) throws IOException {
-    appId.readFields(in);
+    this.applicationId =
+        BuilderUtils.newApplicationId(in.readLong(), in.readInt());
   }
 
   @Override
@@ -67,10 +70,10 @@ public class ClientTokenIdentifier extends TokenIdentifier {
 
   @Override
   public UserGroupInformation getUser() {
-    if (appId == null || "".equals(appId.toString())) {
+    if (this.applicationId == null) {
       return null;
     }
-    return UserGroupInformation.createRemoteUser(appId.toString());
+    return UserGroupInformation.createRemoteUser(this.applicationId.toString());
   }
 
   @InterfaceAudience.Private

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java

@@ -34,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.ipc.RPCUtil;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
 import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore;
@@ -45,6 +44,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRejectedEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 
 /**
@@ -58,14 +58,14 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent> {
   private LinkedList<ApplicationId> completedApps = new LinkedList<ApplicationId>();
 
   private final RMContext rmContext;
-  private final ClientToAMSecretManager clientToAMSecretManager;
+  private final ClientToAMTokenSecretManagerInRM clientToAMSecretManager;
   private final ApplicationMasterService masterService;
   private final YarnScheduler scheduler;
   private final ApplicationACLsManager applicationACLsManager;
   private Configuration conf;
 
   public RMAppManager(RMContext context,
-      ClientToAMSecretManager clientToAMSecretManager,
+      ClientToAMTokenSecretManagerInRM clientToAMSecretManager,
       YarnScheduler scheduler, ApplicationMasterService masterService,
       ApplicationACLsManager applicationACLsManager, Configuration conf) {
     this.rmContext = context;
@@ -230,6 +230,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent> {
     ApplicationId applicationId = submissionContext.getApplicationId();
     RMApp application = null;
     try {
+      // TODO: This needs to move to per-AppAttempt
+      this.clientToAMSecretManager.registerApplication(applicationId);
       String clientTokenStr = null;
       if (UserGroupInformation.isSecurityEnabled()) {
         Token<ClientTokenIdentifier> clientToken = new 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java

@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.event.AsyncDispatcher;
 import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
 import org.apache.hadoop.yarn.server.RMDelegationTokenSecretManager;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
@@ -66,6 +65,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.security.ApplicationTokenSecretManager;
 import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
 import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -96,8 +96,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
   private static final Log LOG = LogFactory.getLog(ResourceManager.class);
   public static final long clusterTimeStamp = System.currentTimeMillis();
 
-  protected ClientToAMSecretManager clientToAMSecretManager =
+  protected ClientToAMTokenSecretManagerInRM clientToAMSecretManager =
-      new ClientToAMSecretManager();
+      new ClientToAMTokenSecretManagerInRM();
   
   protected RMContainerTokenSecretManager containerTokenSecretManager;
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java

@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ContainerManager;
 import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
 import org.apache.hadoop.yarn.api.records.Container;
 import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -54,13 +55,12 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.ApplicationTokenIdentifier;
 import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
-import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.util.ProtoUtils;
 
 /**
@@ -76,7 +76,7 @@ public class AMLauncher implements Runnable {
   private final Configuration conf;
   private final RecordFactory recordFactory = 
       RecordFactoryProvider.getRecordFactory(null);
-  private final ClientToAMSecretManager clientToAMSecretManager;
+  private final ClientToAMTokenSecretManagerInRM clientToAMSecretManager;
   private final AMLauncherEventType eventType;
   private final RMContext rmContext;
   
@@ -85,7 +85,7 @@ public class AMLauncher implements Runnable {
   
   public AMLauncher(RMContext rmContext, RMAppAttempt application,
       AMLauncherEventType eventType,
-      ClientToAMSecretManager clientToAMSecretManager, Configuration conf) {
+      ClientToAMTokenSecretManagerInRM clientToAMSecretManager, Configuration conf) {
     this.application = application;
     this.conf = conf;
     this.clientToAMSecretManager = clientToAMSecretManager;
@@ -194,10 +194,12 @@ public class AMLauncher implements Runnable {
     String parts[] =
         application.getMasterContainer().getNodeHttpAddress().split(":");
     environment.put(ApplicationConstants.NM_HTTP_PORT_ENV, parts[1]);
+    ApplicationId applicationId =
+        application.getAppAttemptId().getApplicationId();
     environment.put(
         ApplicationConstants.APP_SUBMIT_TIME_ENV,
         String.valueOf(rmContext.getRMApps()
-            .get(application.getAppAttemptId().getApplicationId())
+            .get(applicationId)
             .getSubmitTime()));
  
     if (UserGroupInformation.isSecurityEnabled()) {
@@ -237,10 +239,8 @@ public class AMLauncher implements Runnable {
       container.setContainerTokens(
           ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
 
-      ClientTokenIdentifier identifier = new ClientTokenIdentifier(
-          application.getAppAttemptId().getApplicationId());
       SecretKey clientSecretKey =
-          this.clientToAMSecretManager.getMasterKey(identifier);
+          this.clientToAMSecretManager.getMasterKey(applicationId);
       String encoded =
           Base64.encodeBase64URLSafeString(clientSecretKey.getEncoded());
       environment.put(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/ApplicationMasterLauncher.java

@@ -25,9 +25,10 @@ import java.util.concurrent.TimeUnit;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
+import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.service.AbstractService;
 
 
@@ -41,11 +42,11 @@ public class ApplicationMasterLauncher extends AbstractService implements
   private final BlockingQueue<Runnable> masterEvents
     = new LinkedBlockingQueue<Runnable>();
   
-  private ClientToAMSecretManager clientToAMSecretManager;
+  private ClientToAMTokenSecretManagerInRM clientToAMSecretManager;
   protected final RMContext context;
   
   public ApplicationMasterLauncher(
-      ClientToAMSecretManager clientToAMSecretManager, RMContext context) {
+      ClientToAMTokenSecretManagerInRM clientToAMSecretManager, RMContext context) {
     super(ApplicationMasterLauncher.class.getName());
     this.context = context;
     this.launcherPool = new ThreadPoolExecutor(10, 10, 1, 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/ClientToAMTokenSecretManagerInRM.java

@@ -0,0 +1,48 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.security;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.crypto.SecretKey;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.security.client.BaseClientToAMTokenSecretManager;
+
+public class ClientToAMTokenSecretManagerInRM extends
+    BaseClientToAMTokenSecretManager {
+
+  // Per application master-keys for managing client-tokens
+  private Map<ApplicationId, SecretKey> masterKeys =
+      new HashMap<ApplicationId, SecretKey>();
+
+  public synchronized void registerApplication(ApplicationId applicationID) {
+    this.masterKeys.put(applicationID, generateSecret());
+  }
+
+  public synchronized void unRegisterApplication(ApplicationId applicationID) {
+    this.masterKeys.remove(applicationID);
+  }
+
+  @Override
+  public synchronized SecretKey getMasterKey(ApplicationId applicationID) {
+    return this.masterKeys.get(applicationID);
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRMWithCustomAMLauncher.java

@@ -0,0 +1,61 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ContainerManager;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+
+public class MockRMWithCustomAMLauncher extends MockRM {
+
+  private final ContainerManager containerManager;
+
+  public MockRMWithCustomAMLauncher(ContainerManager containerManager) {
+    this(new Configuration(), containerManager);
+  }
+
+  public MockRMWithCustomAMLauncher(Configuration conf,
+      ContainerManager containerManager) {
+    super(conf);
+    this.containerManager = containerManager;
+  }
+
+  @Override
+  protected ApplicationMasterLauncher createAMLauncher() {
+    return new ApplicationMasterLauncher(super.clientToAMSecretManager,
+      getRMContext()) {
+      @Override
+      protected Runnable createRunnableLauncher(RMAppAttempt application,
+          AMLauncherEventType event) {
+        return new AMLauncher(context, application, event,
+          clientToAMSecretManager, getConfig()) {
+          @Override
+          protected ContainerManager getContainerMgrProxy(
+              ContainerId containerId) {
+            return containerManager;
+          }
+        };
+      }
+    };
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAMAuthorization.java

@@ -47,7 +47,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.conf.YarnConfiguration;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
 import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.resourcemanager.TestApplicationMasterLauncher.MockRMWithCustomAMLauncher;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java

@@ -37,7 +37,6 @@ import org.apache.hadoop.yarn.event.Dispatcher;
 import org.apache.hadoop.yarn.event.EventHandler;
 import org.apache.hadoop.yarn.factories.RecordFactory;
 import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
-import org.apache.hadoop.yarn.security.client.ClientToAMSecretManager;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -49,6 +48,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
 import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
 import org.apache.hadoop.yarn.service.Service;
 import org.junit.Test;
@@ -140,7 +140,7 @@ public class TestAppManager{
     }
 
     public TestRMAppManager(RMContext context,
-        ClientToAMSecretManager clientToAMSecretManager,
+        ClientToAMTokenSecretManagerInRM clientToAMSecretManager,
         YarnScheduler scheduler, ApplicationMasterService masterService,
         ApplicationACLsManager applicationACLsManager, Configuration conf) {
       super(context, clientToAMSecretManager, scheduler, masterService,
@@ -342,7 +342,7 @@ public class TestAppManager{
     ApplicationMasterService masterService =
         new ApplicationMasterService(rmContext, scheduler);
     TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
-        new ClientToAMSecretManager(), scheduler, masterService,
+        new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
         new ApplicationACLsManager(conf), conf);
 
     ApplicationId appID = MockApps.newAppID(1);
@@ -390,7 +390,7 @@ public class TestAppManager{
     ApplicationMasterService masterService =
         new ApplicationMasterService(rmContext, scheduler);
     TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
-        new ClientToAMSecretManager(), scheduler, masterService,
+        new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
         new ApplicationACLsManager(conf), conf);
 
     ApplicationId appID = MockApps.newAppID(10);
@@ -438,7 +438,7 @@ public class TestAppManager{
     ApplicationMasterService masterService =
         new ApplicationMasterService(rmContext, scheduler);
     TestRMAppManager appMonitor = new TestRMAppManager(rmContext,
-        new ClientToAMSecretManager(), scheduler, masterService,
+        new ClientToAMTokenSecretManagerInRM(), scheduler, masterService,
         new ApplicationACLsManager(conf), conf);
 
     ApplicationId appID = MockApps.newAppID(0);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterLauncher.java

@@ -22,7 +22,6 @@ import java.util.Map;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.yarn.api.ApplicationConstants;
 import org.apache.hadoop.yarn.api.ContainerManager;
 import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
@@ -35,9 +34,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ContainerId;
 import org.apache.hadoop.yarn.api.records.ContainerState;
 import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncher;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -106,40 +102,6 @@ public class TestApplicationMasterLauncher {
 
   }
 
-  static class MockRMWithCustomAMLauncher extends MockRM {
-
-    private final ContainerManager containerManager;
-
-    public MockRMWithCustomAMLauncher(ContainerManager containerManager) {
-      this(new Configuration(), containerManager);
-    }
-
-    public MockRMWithCustomAMLauncher(Configuration conf,
-        ContainerManager containerManager) {
-      super(conf);
-      this.containerManager = containerManager;
-    }
-
-    @Override
-    protected ApplicationMasterLauncher createAMLauncher() {
-      return new ApplicationMasterLauncher(super.clientToAMSecretManager,
-        getRMContext()) {
-        @Override
-        protected Runnable createRunnableLauncher(RMAppAttempt application,
-            AMLauncherEventType event) {
-          return new AMLauncher(context, application, event,
-            clientToAMSecretManager, getConfig()) {
-            @Override
-            protected ContainerManager getContainerMgrProxy(
-                ContainerId containerId) {
-              return containerManager;
-            }
-          };
-        }
-      };
-    }
-  }
-
   @Test
   public void testAMLaunchAndCleanup() throws Exception {
     Logger rootLogger = LogManager.getRootLogger();

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientTokens.java

@@ -0,0 +1,315 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements.  See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership.  The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License.  You may obtain a copy of the License at
+*
+*     http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.security;
+
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+import junit.framework.Assert;
+
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.KerberosInfo;
+import org.apache.hadoop.security.SecurityInfo;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.security.token.TokenInfo;
+import org.apache.hadoop.security.token.TokenSelector;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ContainerManager;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainerResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
+import org.apache.hadoop.yarn.security.client.ClientTokenIdentifier;
+import org.apache.hadoop.yarn.security.client.ClientTokenSelector;
+import org.apache.hadoop.yarn.server.resourcemanager.ClientRMService;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.junit.Test;
+
+public class TestClientTokens {
+
+  private interface CustomProtocol {
+    public static final long versionID = 1L;
+
+    public void ping();
+  }
+
+  private static class CustomSecurityInfo extends SecurityInfo {
+
+    @Override
+    public TokenInfo getTokenInfo(Class<?> protocol, Configuration conf) {
+      return new TokenInfo() {
+
+        @Override
+        public Class<? extends Annotation> annotationType() {
+          return null;
+        }
+
+        @Override
+        public Class<? extends TokenSelector<? extends TokenIdentifier>>
+            value() {
+          return ClientTokenSelector.class;
+        }
+      };
+    }
+
+    @Override
+    public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
+      return null;
+    }
+  };
+
+  private static class CustomAM extends AbstractService implements
+      CustomProtocol {
+
+    private final ApplicationId appId;
+    private final String secretKey;
+    private InetSocketAddress address;
+    private boolean pinged = false;
+
+    public CustomAM(ApplicationId appId, String secretKeyStr) {
+      super("CustomAM");
+      this.appId = appId;
+      this.secretKey = secretKeyStr;
+    }
+
+    @Override
+    public void ping() {
+      this.pinged = true;
+    }
+
+    @Override
+    public synchronized void start() {
+      Configuration conf = getConfig();
+
+      ClientToAMTokenSecretManager secretManager = null;
+      byte[] bytes = Base64.decodeBase64(this.secretKey);
+      secretManager = new ClientToAMTokenSecretManager(this.appId, bytes);
+      Server server;
+      try {
+        server =
+            new RPC.Builder(conf).setProtocol(CustomProtocol.class)
+              .setNumHandlers(1).setSecretManager(secretManager)
+              .setInstance(this).build();
+      } catch (Exception e) {
+        throw new YarnException(e);
+      }
+      server.start();
+      this.address = NetUtils.getConnectAddress(server);
+      super.start();
+    }
+  }
+
+  private static class CustomNM implements ContainerManager {
+
+    public String clientTokensSecret;
+
+    @Override
+    public StartContainerResponse startContainer(StartContainerRequest request)
+        throws YarnRemoteException {
+      this.clientTokensSecret =
+          request.getContainerLaunchContext().getEnvironment()
+            .get(ApplicationConstants.APPLICATION_CLIENT_SECRET_ENV_NAME);
+      return null;
+    }
+
+    @Override
+    public StopContainerResponse stopContainer(StopContainerRequest request)
+        throws YarnRemoteException {
+      return null;
+    }
+
+    @Override
+    public GetContainerStatusResponse getContainerStatus(
+        GetContainerStatusRequest request) throws YarnRemoteException {
+      return null;
+    }
+
+  }
+
+  @Test
+  public void testClientTokens() throws Exception {
+
+    final Configuration conf = new Configuration();
+    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+      "kerberos");
+    UserGroupInformation.setConfiguration(conf);
+
+    CustomNM containerManager = new CustomNM();
+    final DrainDispatcher dispatcher = new DrainDispatcher();
+
+    MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {
+      protected ClientRMService createClientRMService() {
+        return new ClientRMService(this.rmContext, scheduler,
+          this.rmAppManager, this.applicationACLsManager,
+          this.rmDTSecretManager);
+      };
+
+      @Override
+      protected Dispatcher createDispatcher() {
+        return dispatcher;
+      }
+
+      @Override
+      protected void doSecureLogin() throws IOException {
+      }
+    };
+    rm.start();
+
+    // Submit an app
+    RMApp app = rm.submitApp(1024);
+    dispatcher.await();
+
+    // Set up a node.
+    MockNM nm1 = rm.registerNode("localhost:1234", 3072);
+    nm1.nodeHeartbeat(true);
+    dispatcher.await();
+
+    // Get the app-report.
+    GetApplicationReportRequest request =
+        Records.newRecord(GetApplicationReportRequest.class);
+    request.setApplicationId(app.getApplicationId());
+    GetApplicationReportResponse reportResponse =
+        rm.getClientRMService().getApplicationReport(request);
+    ApplicationReport appReport = reportResponse.getApplicationReport();
+    String clientTokenEncoded = appReport.getClientToken();
+
+    // Wait till AM is 'launched'
+    int waitTime = 0;
+    while (containerManager.clientTokensSecret == null && waitTime++ < 20) {
+      Thread.sleep(1000);
+    }
+    Assert.assertNotNull(containerManager.clientTokensSecret);
+
+    // Start the AM with the correct shared-secret.
+    final CustomAM am =
+        new CustomAM(app.getApplicationId(),
+          containerManager.clientTokensSecret);
+    am.init(conf);
+    am.start();
+
+    // Now the real test!
+    // Set up clients to be able to pick up correct tokens.
+    SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
+
+    // Verify denial for unauthenticated user
+    try {
+      CustomProtocol client =
+          (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
+            conf);
+      client.ping();
+      fail("Access by unauthenticated user should fail!!");
+    } catch (Exception e) {
+      Assert.assertFalse(am.pinged);
+    }
+
+    // Verify denial for a malicious user
+    UserGroupInformation ugi = UserGroupInformation.createRemoteUser("me");
+    Token<ClientTokenIdentifier> clientToken =
+        new Token<ClientTokenIdentifier>();
+    clientToken.decodeFromUrlString(clientTokenEncoded);
+    // RPC layer client expects ip:port as service for tokens
+    SecurityUtil.setTokenService(clientToken, am.address);
+
+    // Malicious user, messes with appId
+    ClientTokenIdentifier maliciousID =
+        new ClientTokenIdentifier(BuilderUtils.newApplicationId(app
+          .getApplicationId().getClusterTimestamp(), 42));
+
+    Token<ClientTokenIdentifier> maliciousToken =
+        new Token<ClientTokenIdentifier>(maliciousID.getBytes(),
+          clientToken.getPassword(), clientToken.getKind(),
+          clientToken.getService());
+    ugi.addToken(maliciousToken);
+
+    try {
+      ugi.doAs(new PrivilegedExceptionAction<Void>() {
+        @Override
+        public Void run() throws Exception {
+          CustomProtocol client =
+              (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L,
+                am.address, conf);
+          client.ping();
+          fail("Connection initiation with illegally modified "
+              + "tokens is expected to fail.");
+          return null;
+        }
+      });
+    } catch (YarnRemoteException e) {
+      fail("Cannot get a YARN remote exception as "
+          + "it will indicate RPC success");
+    } catch (Exception e) {
+      Assert
+        .assertEquals(java.lang.reflect.UndeclaredThrowableException.class
+          .getCanonicalName(), e.getClass().getCanonicalName());
+      Assert.assertTrue(e
+        .getCause()
+        .getMessage()
+        .contains(
+          "DIGEST-MD5: digest response format violation. "
+              + "Mismatched response."));
+      Assert.assertFalse(am.pinged);
+    }
+
+    // Now for an authenticated user
+    ugi = UserGroupInformation.createRemoteUser("me");
+    ugi.addToken(clientToken);
+
+    ugi.doAs(new PrivilegedExceptionAction<Void>() {
+      @Override
+      public Void run() throws Exception {
+        CustomProtocol client =
+            (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
+              conf);
+        client.ping();
+        Assert.assertTrue(am.pinged);
+        return null;
+      }
+    });
+  }
+
+}