diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 2cbfba52e5e..a976bc4a8d1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -44,6 +44,9 @@ Release 2.7.4 - UNRELEASED YARN-4573. Fix test failure in TestRMAppTransitions#testAppRunningKill and testAppKilledKilled. (Takashi Ohnishi via rohithsharmaks) + YARN-5382. RM does not audit log kill request for active applications + (Vrushali C via jlowe) + Release 2.7.3 - UNRELEASED INCOMPATIBLE CHANGES diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 6ef69cc010a..2ee2ba756b4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.InputStream; +import java.net.InetAddress; import java.net.InetSocketAddress; import java.security.AccessControlException; import java.text.MessageFormat; @@ -129,8 +130,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem; import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent; -import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType; +import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppKillByClientEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; @@ -638,14 +638,18 @@ public KillApplicationResponse forceKillApplication( } if (application.isAppFinalStateStored()) { - RMAuditLogger.logSuccess(callerUGI.getShortUserName(), - AuditConstants.KILL_APP_REQUEST, "ClientRMService", applicationId); return KillApplicationResponse.newInstance(true); } - this.rmContext.getDispatcher().getEventHandler().handle( - new RMAppEvent(applicationId, RMAppEventType.KILL, - "Application killed by user.")); + String message = "Kill application " + applicationId + " received from " + + callerUGI; + InetAddress remoteAddress = Server.getRemoteIp(); + if (null != remoteAddress) { + message += " at " + remoteAddress.getHostAddress(); + } + this.rmContext.getDispatcher().getEventHandler() + .handle(new RMAppKillByClientEvent(applicationId, message, callerUGI, + remoteAddress)); // For UnmanagedAMs, return true so they don't retry return KillApplicationResponse.newInstance( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index db8a46a0d4d..d61221021a9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -68,10 +68,13 @@ public static class AuditConstants { * A helper api for creating an audit log for a successful event. */ static String createSuccessLog(String user, String operation, String target, - ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId, InetAddress ip) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); - addRemoteIP(b); + if (ip != null) { + add(Keys.IP, ip.getHostAddress(), b); + } add(Keys.OPERATION, operation, b); add(Keys.TARGET, target ,b); add(Keys.RESULT, AuditConstants.SUCCESS, b); @@ -87,6 +90,13 @@ static String createSuccessLog(String user, String operation, String target, return b.toString(); } + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId) { + return createSuccessLog(user, operation, target, appId, attemptId, + containerId, Server.getRemoteIp()); + } + /** * Create a readable and parseable audit log string for a successful event. * @@ -166,6 +176,34 @@ public static void logSuccess(String user, String operation, String target) { } } + /** + * Create a readable and parseable audit log string for a successful event. + * + * @param user + * User who made the service request to the ResourceManager. + * @param operation + * Operation requested by the user. + * @param target + * The target on which the operation is being performed. + * @param appId + * Application Id in which operation was performed. + * @param ip + * The ip address of the caller. + * + *
+ *
+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val + * delimiter and hence the value fields should not contains tabs + * ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + ApplicationId appId, InetAddress ip) { + if (LOG.isInfoEnabled()) { + LOG.info( + createSuccessLog(user, operation, target, appId, null, null, ip)); + } + } + /** * A helper api for creating an audit log for a failure event. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java index 0ac83310aa2..f13e09d01c5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp; import java.io.IOException; +import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; import java.nio.ByteBuffer; @@ -64,8 +65,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent; import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable; import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData; @@ -713,7 +716,7 @@ public void handle(RMAppEvent event) { if (oldState != getState()) { LOG.info(appID + " State change from " + oldState + " to " - + getState()); + + getState() + " on event=" + event.getType()); } } finally { this.writeLock.unlock(); @@ -1092,6 +1095,23 @@ public void transition(RMAppImpl app, RMAppEvent event) { }; } + /** + * Log the audit event for kill by client. + * @param event The {@link RMAppEvent} to be logged + */ + static void auditLogKillEvent(RMAppEvent event) { + if (event instanceof RMAppKillByClientEvent) { + RMAppKillByClientEvent killEvent = (RMAppKillByClientEvent) event; + UserGroupInformation callerUGI = killEvent.getCallerUGI(); + String userName = null; + if (callerUGI != null) { + userName = callerUGI.getShortUserName(); + } + InetAddress remoteIP = killEvent.getIp(); + RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST, + "RMAppImpl", event.getApplicationId(), remoteIP); + } + } private static class AppKilledTransition extends FinalTransition { public AppKilledTransition() { @@ -1102,6 +1122,7 @@ public AppKilledTransition() { public void transition(RMAppImpl app, RMAppEvent event) { app.diagnostics.append(event.getDiagnosticMsg()); super.transition(app, event); + RMAppImpl.auditLogKillEvent(event); }; } @@ -1115,6 +1136,7 @@ public void transition(RMAppImpl app, RMAppEvent event) { app.handler.handle( new RMAppAttemptEvent(app.currentAttempt.getAppAttemptId(), RMAppAttemptEventType.KILL, event.getDiagnosticMsg())); + RMAppImpl.auditLogKillEvent(event); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillByClientEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillByClientEvent.java new file mode 100644 index 00000000000..03300872aad --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillByClientEvent.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.rmapp; + +import java.net.InetAddress; + +import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.yarn.api.records.ApplicationId; + +/** + * An event class that is used to help with logging information + * when an application KILL event is needed. + * + */ +public class RMAppKillByClientEvent extends RMAppEvent { + + private final UserGroupInformation callerUGI; + private final InetAddress ip; + + /** + * constructor to create an event used for logging during user driven kill + * invocations. + * + * @param appId application id + * @param diagnostics message about the kill event + * @param callerUGI caller's user and group information + * @param remoteIP ip address of the caller + */ + public RMAppKillByClientEvent(ApplicationId appId, String diagnostics, + UserGroupInformation callerUGI, InetAddress remoteIP) { + super(appId, RMAppEventType.KILL, diagnostics); + this.callerUGI = callerUGI; + this.ip = remoteIP; + } + + /** + * returns the {@link UserGroupInformation} information. + * @return UserGroupInformation + */ + public final UserGroupInformation getCallerUGI() { + return callerUGI; + } + + /** + * returns the ip address stored in this event. + * @return remoteIP + */ + public final InetAddress getIp() { + return ip; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 41776cce3a5..0954a1ac3f3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -18,11 +18,13 @@ package org.apache.hadoop.yarn.server.resourcemanager; import static org.junit.Assert.assertEquals; + import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; import java.net.InetAddress; import java.net.InetSocketAddress; +import java.net.UnknownHostException; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.ipc.RPC; @@ -36,7 +38,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys; import org.junit.Before; import org.junit.Test; - +import org.junit.Assert; /** * Tests {@link RMAuditLogger}. @@ -92,14 +94,26 @@ public void testKeyValLogFormat() throws Exception { * Test the AuditLog format for successful events. */ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId) { - String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, - appId, attemptId, containerId); + ApplicationAttemptId attemptId, ContainerId containerId, + InetAddress remoteIp) { + InetAddress ip; + if (remoteIp != null) { + ip = remoteIp; + } else { + ip = Server.getRemoteIp(); + } + StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); + + String sLog; if (checkIP) { - InetAddress ip = Server.getRemoteIp(); + sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, + attemptId, containerId, ip); expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t"); + } else { + sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, + attemptId, containerId); } expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS"); @@ -115,6 +129,11 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, assertEquals(expLog.toString(), sLog); } + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId) { + testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null); + } + /** * Test the AuditLog format for successful events passing nulls. */ @@ -131,6 +150,39 @@ private void testSuccessLogNulls(boolean checkIP) { assertEquals(expLog.toString(), sLog); } + private void testSuccessLogFormatHelperWithIP(boolean checkIP, + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId, InetAddress ip) { + testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, ip); + } + + /** + * Tests the SuccessLog with two IP addresses. + * + * @param checkIP + * @param appId + * @param attemptId + * @param containerId + */ + private void testSuccessLogFormatHelperWithIP(boolean checkIP, + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId) { + testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID, + InetAddress.getLoopbackAddress()); + byte[] ipAddr = new byte[] { 100, 10, 10, 1 }; + + InetAddress addr = null; + try { + addr = InetAddress.getByAddress(ipAddr); + } catch (UnknownHostException uhe) { + // should not happen as long as IP address format + // stays the same + Assert.fail("Check ip address being constructed"); + } + testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID, + addr); + } + /** * Test the AuditLog format for successful events with the various * parameters. @@ -144,6 +196,7 @@ private void testSuccessLogFormat(boolean checkIP) { testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID); testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID); testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); + testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID); testSuccessLogNulls(checkIP); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java index bbd205b5211..dce1f53ec94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java @@ -38,6 +38,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.DataOutputBuffer; +import org.apache.hadoop.ipc.Server; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; @@ -548,11 +549,14 @@ public void testAppRecoverPath() throws IOException { public void testAppNewKill() throws IOException { LOG.info("--- START: testAppNewKill ---"); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppNewKill", new String[] { "foo_group" }); RMApp application = createNewTestApp(null); // NEW => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + + RMAppEvent event = new RMAppKillByClientEvent( + application.getApplicationId(), "Application killed by user.", fooUser, + Server.getRemoteIp()); application.handle(event); rmDispatcher.await(); sendAppUpdateSavedEvent(application); @@ -601,11 +605,15 @@ public void testAppNewRejectAddToStore() throws IOException { public void testAppNewSavingKill() throws IOException { LOG.info("--- START: testAppNewSavingKill ---"); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppNewSavingKill", new String[] { "foo_group" }); + RMApp application = testCreateAppNewSaving(null); // NEW_SAVING => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + RMAppEvent event = new RMAppKillByClientEvent( + application.getApplicationId(), "Application killed by user.", fooUser, + Server.getRemoteIp()); + application.handle(event); rmDispatcher.await(); sendAppUpdateSavedEvent(application); @@ -651,11 +659,13 @@ public void testAppSubmittedRejected() throws IOException { @Test public void testAppSubmittedKill() throws IOException, InterruptedException { LOG.info("--- START: testAppSubmittedKill---"); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppSubmittedKill", new String[] { "foo_group" }); RMApp application = testCreateAppSubmittedNoRecovery(null); // SUBMITTED => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + RMAppEvent event = new RMAppKillByClientEvent( + application.getApplicationId(), "Application killed by user.", fooUser, + Server.getRemoteIp()); application.handle(event); rmDispatcher.await(); sendAppUpdateSavedEvent(application); @@ -703,11 +713,13 @@ public void testAppAcceptedFailed() throws IOException { @Test public void testAppAcceptedKill() throws IOException, InterruptedException { LOG.info("--- START: testAppAcceptedKill ---"); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppAcceptedKill", new String[] { "foo_group" }); RMApp application = testCreateAppAccepted(null); // ACCEPTED => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + RMAppEvent event = new RMAppKillByClientEvent( + application.getApplicationId(), "Application killed by user.", fooUser, + Server.getRemoteIp()); application.handle(event); rmDispatcher.await(); @@ -750,12 +762,14 @@ public void testAppAcceptedAttemptKilled() throws IOException, @Test public void testAppRunningKill() throws IOException { LOG.info("--- START: testAppRunningKill ---"); + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppRunningKill", new String[] { "foo_group" }); RMApp application = testCreateAppRunning(null); // RUNNING => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + RMAppEvent event = new RMAppKillByClientEvent( + application.getApplicationId(), "Application killed by user.", fooUser, + Server.getRemoteIp()); application.handle(event); rmDispatcher.await(); @@ -915,13 +929,14 @@ public void testAppFailedFailed() throws IOException { @Test (timeout = 30000) public void testAppKilledKilled() throws IOException { LOG.info("--- START: testAppKilledKilled ---"); - + UserGroupInformation fooUser = UserGroupInformation.createUserForTesting( + "fooTestAppKilledKill", new String[] { "foo_group" }); RMApp application = testCreateAppRunning(null); // RUNNING => KILLED event RMAppEventType.KILL - RMAppEvent event = - new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL, - "Application killed by user."); + RMAppEvent event = new RMAppKillByClientEvent( + application.getApplicationId(), "Application killed by user.", fooUser, + Server.getRemoteIp()); application.handle(event); rmDispatcher.await(); sendAttemptUpdateSavedEvent(application);