YARN-5382. RM does not audit log kill request for active applications. Contributed by Vrushali C

hadoop-yarn-project/CHANGES.txt

@@ -44,6 +44,9 @@ Release 2.7.4 - UNRELEASED
     YARN-4573. Fix test failure in TestRMAppTransitions#testAppRunningKill and
     testAppKilledKilled. (Takashi Ohnishi via rohithsharmaks)
 
+    YARN-5382. RM does not audit log kill request for active applications
+    (Vrushali C via jlowe)
+
 Release 2.7.3 - UNRELEASED
 
   INCOMPATIBLE CHANGES

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -20,6 +20,7 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.InetAddress;
 import java.net.InetSocketAddress;
 import java.security.AccessControlException;
 import java.text.MessageFormat;
@@ -129,8 +130,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppKillByClientEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
 import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -638,14 +638,18 @@ public KillApplicationResponse forceKillApplication(
     }
 
     if (application.isAppFinalStateStored()) {
-      RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
-          AuditConstants.KILL_APP_REQUEST, "ClientRMService", applicationId);
       return KillApplicationResponse.newInstance(true);
     }
 
-    this.rmContext.getDispatcher().getEventHandler().handle(
+    String message = "Kill application " + applicationId + " received from "
-        new RMAppEvent(applicationId, RMAppEventType.KILL,
+        + callerUGI;
-        "Application killed by user."));
+    InetAddress remoteAddress = Server.getRemoteIp();
+    if (null != remoteAddress) {
+      message += " at " + remoteAddress.getHostAddress();
+    }
+    this.rmContext.getDispatcher().getEventHandler()
+        .handle(new RMAppKillByClientEvent(applicationId, message, callerUGI,
+            remoteAddress));
 
     // For UnmanagedAMs, return true so they don't retry
     return KillApplicationResponse.newInstance(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java

@@ -68,10 +68,13 @@ public static class AuditConstants {
    * A helper api for creating an audit log for a successful event.
    */
   static String createSuccessLog(String user, String operation, String target,
-      ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) {
+      ApplicationId appId, ApplicationAttemptId attemptId,
+      ContainerId containerId, InetAddress ip) {
     StringBuilder b = new StringBuilder();
     start(Keys.USER, user, b);
-    addRemoteIP(b);
+    if (ip != null) {
+      add(Keys.IP, ip.getHostAddress(), b);
+    }
     add(Keys.OPERATION, operation, b);
     add(Keys.TARGET, target ,b);
     add(Keys.RESULT, AuditConstants.SUCCESS, b);
@@ -87,6 +90,13 @@ static String createSuccessLog(String user, String operation, String target,
     return b.toString();
   }
 
+  static String createSuccessLog(String user, String operation, String target,
+      ApplicationId appId, ApplicationAttemptId attemptId,
+      ContainerId containerId) {
+    return createSuccessLog(user, operation, target, appId, attemptId,
+        containerId, Server.getRemoteIp());
+  }
+
   /**
    * Create a readable and parseable audit log string for a successful event.
    *
@@ -166,6 +176,34 @@ public static void logSuccess(String user, String operation, String target) {
     }
   }
 
+  /**
+   * Create a readable and parseable audit log string for a successful event.
+   *
+   * @param user
+   *          User who made the service request to the ResourceManager.
+   * @param operation
+   *          Operation requested by the user.
+   * @param target
+   *          The target on which the operation is being performed.
+   * @param appId
+   *          Application Id in which operation was performed.
+   * @param ip
+   *          The ip address of the caller.
+   *
+   *          <br>
+   *          <br>
+   *          Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val
+   *          delimiter and hence the value fields should not contains tabs
+   *          ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ApplicationId appId, InetAddress ip) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(
+          createSuccessLog(user, operation, target, appId, null, null, ip));
+    }
+  }
+
   /**
    * A helper api for creating an audit log for a failure event.
    */

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java

@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
 
 import java.io.IOException;
+import java.net.InetAddress;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.ByteBuffer;
@@ -64,8 +65,10 @@
 import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
 import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
 import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
@@ -713,7 +716,7 @@ public void handle(RMAppEvent event) {
 
       if (oldState != getState()) {
         LOG.info(appID + " State change from " + oldState + " to "
-            + getState());
+            + getState() + " on event=" + event.getType());
       }
     } finally {
       this.writeLock.unlock();
@@ -1092,6 +1095,23 @@ public void transition(RMAppImpl app, RMAppEvent event) {
     };
   }
 
+  /**
+   * Log the audit event for kill by client.
+   * @param event The {@link RMAppEvent} to be logged
+   */
+  static void auditLogKillEvent(RMAppEvent event) {
+    if (event instanceof RMAppKillByClientEvent) {
+      RMAppKillByClientEvent killEvent = (RMAppKillByClientEvent) event;
+      UserGroupInformation callerUGI = killEvent.getCallerUGI();
+      String userName = null;
+      if (callerUGI != null) {
+        userName = callerUGI.getShortUserName();
+      }
+      InetAddress remoteIP = killEvent.getIp();
+      RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST,
+          "RMAppImpl", event.getApplicationId(), remoteIP);
+    }
+  }
 
   private static class AppKilledTransition extends FinalTransition {
     public AppKilledTransition() {
@@ -1102,6 +1122,7 @@ public AppKilledTransition() {
     public void transition(RMAppImpl app, RMAppEvent event) {
       app.diagnostics.append(event.getDiagnosticMsg());
       super.transition(app, event);
+      RMAppImpl.auditLogKillEvent(event);
     };
   }
 
@@ -1115,6 +1136,7 @@ public void transition(RMAppImpl app, RMAppEvent event) {
       app.handler.handle(
           new RMAppAttemptEvent(app.currentAttempt.getAppAttemptId(),
               RMAppAttemptEventType.KILL, event.getDiagnosticMsg()));
+      RMAppImpl.auditLogKillEvent(event);
     }
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillByClientEvent.java

@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
+
+import java.net.InetAddress;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+/**
+ * An event class that is used to help with logging information
+ * when an application KILL event is needed.
+ *
+ */
+public class RMAppKillByClientEvent extends RMAppEvent {
+
+  private final UserGroupInformation callerUGI;
+  private final InetAddress ip;
+
+  /**
+   * constructor to create an event used for logging during user driven kill
+   * invocations.
+   *
+   * @param appId application id
+   * @param diagnostics message about the kill event
+   * @param callerUGI caller's user and group information
+   * @param remoteIP ip address of the caller
+   */
+  public RMAppKillByClientEvent(ApplicationId appId, String diagnostics,
+      UserGroupInformation callerUGI, InetAddress remoteIP) {
+    super(appId, RMAppEventType.KILL, diagnostics);
+    this.callerUGI = callerUGI;
+    this.ip = remoteIP;
+  }
+
+  /**
+   * returns the {@link UserGroupInformation} information.
+   * @return UserGroupInformation
+   */
+  public final UserGroupInformation getCallerUGI() {
+    return callerUGI;
+  }
+
+  /**
+   * returns the ip address stored in this event.
+   * @return remoteIP
+   */
+  public final InetAddress getIp() {
+    return ip;
+  }
+}

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java

@@ -18,11 +18,13 @@
 package org.apache.hadoop.yarn.server.resourcemanager;
 
 import static org.junit.Assert.assertEquals;
+
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.when;
 
 import java.net.InetAddress;
 import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RPC;
@@ -36,7 +38,7 @@
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
 import org.junit.Before;
 import org.junit.Test;
-
+import org.junit.Assert;
 
 /**
  * Tests {@link RMAuditLogger}.
@@ -92,14 +94,26 @@ public void testKeyValLogFormat() throws Exception {
    * Test the AuditLog format for successful events.
    */
   private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
-      ApplicationAttemptId attemptId, ContainerId containerId) {
+      ApplicationAttemptId attemptId, ContainerId containerId,
-    String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
+      InetAddress remoteIp) {
-        appId, attemptId, containerId);
+    InetAddress ip;
+    if (remoteIp != null) {
+      ip = remoteIp;
+    } else {
+      ip = Server.getRemoteIp();
+    }
+
     StringBuilder expLog = new StringBuilder();
     expLog.append("USER=test\t");
+
+    String sLog;
     if (checkIP) {
-      InetAddress ip = Server.getRemoteIp();
+      sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
+          attemptId, containerId, ip);
       expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
+    } else {
+      sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
+          attemptId, containerId);
     }
     expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
 
@@ -115,6 +129,11 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
     assertEquals(expLog.toString(), sLog);
   }
 
+  private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+      ApplicationAttemptId attemptId, ContainerId containerId) {
+    testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null);
+  }
+
   /**
    * Test the AuditLog format for successful events passing nulls.
    */
@@ -131,6 +150,39 @@ private void testSuccessLogNulls(boolean checkIP) {
     assertEquals(expLog.toString(), sLog);
   }
 
+  private void testSuccessLogFormatHelperWithIP(boolean checkIP,
+      ApplicationId appId, ApplicationAttemptId attemptId,
+      ContainerId containerId, InetAddress ip) {
+    testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, ip);
+  }
+
+  /**
+   * Tests the SuccessLog with two IP addresses.
+   *
+   * @param checkIP
+   * @param appId
+   * @param attemptId
+   * @param containerId
+   */
+  private void testSuccessLogFormatHelperWithIP(boolean checkIP,
+      ApplicationId appId, ApplicationAttemptId attemptId,
+      ContainerId containerId) {
+    testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID,
+        InetAddress.getLoopbackAddress());
+    byte[] ipAddr = new byte[] { 100, 10, 10, 1 };
+
+    InetAddress addr = null;
+    try {
+      addr = InetAddress.getByAddress(ipAddr);
+    } catch (UnknownHostException uhe) {
+      // should not happen as long as IP address format
+      // stays the same
+      Assert.fail("Check ip address being constructed");
+    }
+    testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID,
+        addr);
+  }
+
   /**
    * Test the AuditLog format for successful events with the various
    * parameters.
@@ -144,6 +196,7 @@ private void testSuccessLogFormat(boolean checkIP) {
     testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID);
     testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
     testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
+    testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
     testSuccessLogNulls(checkIP);
   }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java

@@ -38,6 +38,7 @@
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.ipc.Server;
 import org.apache.hadoop.security.Credentials;
 import org.apache.hadoop.security.SecurityUtil;
 import org.apache.hadoop.security.UserGroupInformation;
@@ -548,11 +549,14 @@ public void testAppRecoverPath() throws IOException {
   public void testAppNewKill() throws IOException {
     LOG.info("--- START: testAppNewKill ---");
 
+    UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
+        "fooTestAppNewKill", new String[] { "foo_group" });
     RMApp application = createNewTestApp(null);
     // NEW => KILLED event RMAppEventType.KILL
-    RMAppEvent event =
+
-        new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
+    RMAppEvent event = new RMAppKillByClientEvent(
-        "Application killed by user.");
+        application.getApplicationId(), "Application killed by user.", fooUser,
+        Server.getRemoteIp());
     application.handle(event);
     rmDispatcher.await();
     sendAppUpdateSavedEvent(application);
@@ -601,11 +605,15 @@ public void testAppNewRejectAddToStore() throws IOException {
   public void testAppNewSavingKill() throws IOException {
     LOG.info("--- START: testAppNewSavingKill ---");
 
+    UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
+        "fooTestAppNewSavingKill", new String[] { "foo_group" });
+
     RMApp application = testCreateAppNewSaving(null);
     // NEW_SAVING => KILLED event RMAppEventType.KILL
-    RMAppEvent event =
+    RMAppEvent event = new RMAppKillByClientEvent(
-        new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
+        application.getApplicationId(), "Application killed by user.", fooUser,
-        "Application killed by user.");
+        Server.getRemoteIp());
+
     application.handle(event);
     rmDispatcher.await();
     sendAppUpdateSavedEvent(application);
@@ -651,11 +659,13 @@ public void testAppSubmittedRejected() throws IOException {
   @Test
   public void testAppSubmittedKill() throws IOException, InterruptedException {
     LOG.info("--- START: testAppSubmittedKill---");
+    UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
+        "fooTestAppSubmittedKill", new String[] { "foo_group" });
     RMApp application = testCreateAppSubmittedNoRecovery(null);
     // SUBMITTED => KILLED event RMAppEventType.KILL
-    RMAppEvent event =
+    RMAppEvent event = new RMAppKillByClientEvent(
-        new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
+        application.getApplicationId(), "Application killed by user.", fooUser,
-        "Application killed by user.");
+        Server.getRemoteIp());
     application.handle(event);
     rmDispatcher.await();
     sendAppUpdateSavedEvent(application);
@@ -703,11 +713,13 @@ public void testAppAcceptedFailed() throws IOException {
   @Test
   public void testAppAcceptedKill() throws IOException, InterruptedException {
     LOG.info("--- START: testAppAcceptedKill ---");
+    UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
+        "fooTestAppAcceptedKill", new String[] { "foo_group" });
     RMApp application = testCreateAppAccepted(null);
     // ACCEPTED => KILLED event RMAppEventType.KILL
-    RMAppEvent event =
+    RMAppEvent event = new RMAppKillByClientEvent(
-        new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
+        application.getApplicationId(), "Application killed by user.", fooUser,
-        "Application killed by user.");
+        Server.getRemoteIp());
     application.handle(event);
     rmDispatcher.await();
 
@@ -750,12 +762,14 @@ public void testAppAcceptedAttemptKilled() throws IOException,
   @Test
   public void testAppRunningKill() throws IOException {
     LOG.info("--- START: testAppRunningKill ---");
+    UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
+        "fooTestAppRunningKill", new String[] { "foo_group" });
 
     RMApp application = testCreateAppRunning(null);
     // RUNNING => KILLED event RMAppEventType.KILL
-    RMAppEvent event =
+    RMAppEvent event = new RMAppKillByClientEvent(
-        new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
+        application.getApplicationId(), "Application killed by user.", fooUser,
-        "Application killed by user.");
+        Server.getRemoteIp());
     application.handle(event);
     rmDispatcher.await();
 
@@ -915,13 +929,14 @@ public void testAppFailedFailed() throws IOException {
   @Test (timeout = 30000)
   public void testAppKilledKilled() throws IOException {
     LOG.info("--- START: testAppKilledKilled ---");
-
+    UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
+        "fooTestAppKilledKill", new String[] { "foo_group" });
     RMApp application = testCreateAppRunning(null);
 
     // RUNNING => KILLED event RMAppEventType.KILL
-    RMAppEvent event =
+    RMAppEvent event = new RMAppKillByClientEvent(
-        new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
+        application.getApplicationId(), "Application killed by user.", fooUser,
-        "Application killed by user.");
+        Server.getRemoteIp());
     application.handle(event);
     rmDispatcher.await();
     sendAttemptUpdateSavedEvent(application);