Apache/hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5382. RM does not audit log kill request for active applications. Contributed by Vrushali C

Browse Source
This commit is contained in:
Jason Lowe 2016-08-10 19:18:02 +00:00
parent e88546596d
commit c2d3af39f1
7 changed files with 236 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
hadoop-yarn-project/CHANGES.txt
View File

@ -44,6 +44,9 @@ Release 2.7.4 - UNRELEASED
YARN-4573. Fix test failure in TestRMAppTransitions#testAppRunningKill and
testAppKilledKilled. (Takashi Ohnishi via rohithsharmaks)
YARN-5382. RM does not audit log kill request for active applications
(Vrushali C via jlowe)
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES

18
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -20,6 +20,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.AccessControlException;
import java.text.MessageFormat;
@ -129,8 +130,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppKillByClientEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMoveEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@ -638,14 +638,18 @@ public KillApplicationResponse forceKillApplication(
}
if (application.isAppFinalStateStored()) {
RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
AuditConstants.KILL_APP_REQUEST, "ClientRMService", applicationId);
return KillApplicationResponse.newInstance(true);
}
this.rmContext.getDispatcher().getEventHandler().handle(
new RMAppEvent(applicationId, RMAppEventType.KILL,
"Application killed by user."));
String message = "Kill application " + applicationId + " received from "
+ callerUGI;
InetAddress remoteAddress = Server.getRemoteIp();
if (null != remoteAddress) {
message += " at " + remoteAddress.getHostAddress();
}
this.rmContext.getDispatcher().getEventHandler()
.handle(new RMAppKillByClientEvent(applicationId, message, callerUGI,
remoteAddress));
// For UnmanagedAMs, return true so they don't retry
return KillApplicationResponse.newInstance(

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
View File

@ -68,10 +68,13 @@ public static class AuditConstants {
* A helper api for creating an audit log for a successful event.
*/
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) {
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, InetAddress ip) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
if (ip != null) {
add(Keys.IP, ip.getHostAddress(), b);
}
add(Keys.OPERATION, operation, b);
add(Keys.TARGET, target ,b);
add(Keys.RESULT, AuditConstants.SUCCESS, b);
@ -87,6 +90,13 @@ static String createSuccessLog(String user, String operation, String target,
return b.toString();
}
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId) {
return createSuccessLog(user, operation, target, appId, attemptId,
containerId, Server.getRemoteIp());
}
/**
* Create a readable and parseable audit log string for a successful event.
*
@ -166,6 +176,34 @@ public static void logSuccess(String user, String operation, String target) {
}
}
/**
* Create a readable and parseable audit log string for a successful event.
*
* @param user
* User who made the service request to the ResourceManager.
* @param operation
* Operation requested by the user.
* @param target
* The target on which the operation is being performed.
* @param appId
* Application Id in which operation was performed.
* @param ip
* The ip address of the caller.
*
* <br>
* <br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val
* delimiter and hence the value fields should not contains tabs
* ('\t').
*/
public static void logSuccess(String user, String operation, String target,
ApplicationId appId, InetAddress ip) {
if (LOG.isInfoEnabled()) {
LOG.info(
createSuccessLog(user, operation, target, appId, null, null, ip));
}
}
/**
* A helper api for creating an audit log for a failure event.
*/

24
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
View File

@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
@ -64,8 +65,10 @@
import org.apache.hadoop.yarn.server.resourcemanager.ApplicationMasterService;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
@ -713,7 +716,7 @@ public void handle(RMAppEvent event) {
if (oldState != getState()) {
LOG.info(appID + " State change from " + oldState + " to "
+ getState());
+ getState() + " on event=" + event.getType());
}
} finally {
this.writeLock.unlock();
@ -1092,6 +1095,23 @@ public void transition(RMAppImpl app, RMAppEvent event) {
};
}
/**
* Log the audit event for kill by client.
* @param event The {@link RMAppEvent} to be logged
*/
static void auditLogKillEvent(RMAppEvent event) {
if (event instanceof RMAppKillByClientEvent) {
RMAppKillByClientEvent killEvent = (RMAppKillByClientEvent) event;
UserGroupInformation callerUGI = killEvent.getCallerUGI();
String userName = null;
if (callerUGI != null) {
userName = callerUGI.getShortUserName();
}
InetAddress remoteIP = killEvent.getIp();
RMAuditLogger.logSuccess(userName, AuditConstants.KILL_APP_REQUEST,
"RMAppImpl", event.getApplicationId(), remoteIP);
}
}
private static class AppKilledTransition extends FinalTransition {
public AppKilledTransition() {
@ -1102,6 +1122,7 @@ public AppKilledTransition() {
public void transition(RMAppImpl app, RMAppEvent event) {
app.diagnostics.append(event.getDiagnosticMsg());
super.transition(app, event);
RMAppImpl.auditLogKillEvent(event);
};
}
@ -1115,6 +1136,7 @@ public void transition(RMAppImpl app, RMAppEvent event) {
app.handler.handle(
new RMAppAttemptEvent(app.currentAttempt.getAppAttemptId(),
RMAppAttemptEventType.KILL, event.getDiagnosticMsg()));
RMAppImpl.auditLogKillEvent(event);
}
}

67
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppKillByClientEvent.java Normal file
View File

@ -0,0 +1,67 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import java.net.InetAddress;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
/**
* An event class that is used to help with logging information
* when an application KILL event is needed.
*
*/
public class RMAppKillByClientEvent extends RMAppEvent {
private final UserGroupInformation callerUGI;
private final InetAddress ip;
/**
* constructor to create an event used for logging during user driven kill
* invocations.
*
* @param appId application id
* @param diagnostics message about the kill event
* @param callerUGI caller's user and group information
* @param remoteIP ip address of the caller
*/
public RMAppKillByClientEvent(ApplicationId appId, String diagnostics,
UserGroupInformation callerUGI, InetAddress remoteIP) {
super(appId, RMAppEventType.KILL, diagnostics);
this.callerUGI = callerUGI;
this.ip = remoteIP;
}
/**
* returns the {@link UserGroupInformation} information.
* @return UserGroupInformation
*/
public final UserGroupInformation getCallerUGI() {
return callerUGI;
}
/**
* returns the ip address stored in this event.
* @return remoteIP
*/
public final InetAddress getIp() {
return ip;
}
}

63
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
View File

@ -18,11 +18,13 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
@ -36,7 +38,7 @@
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
import org.junit.Before;
import org.junit.Test;
import org.junit.Assert;
/**
* Tests {@link RMAuditLogger}.
@ -92,14 +94,26 @@ public void testKeyValLogFormat() throws Exception {
* Test the AuditLog format for successful events.
*/
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
appId, attemptId, containerId);
ApplicationAttemptId attemptId, ContainerId containerId,
InetAddress remoteIp) {
InetAddress ip;
if (remoteIp != null) {
ip = remoteIp;
} else {
ip = Server.getRemoteIp();
}
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
String sLog;
if (checkIP) {
InetAddress ip = Server.getRemoteIp();
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
attemptId, containerId, ip);
expLog.append(Keys.IP.name() + "=" + ip.getHostAddress() + "\t");
} else {
sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
attemptId, containerId);
}
expLog.append("OPERATION=oper\tTARGET=tgt\tRESULT=SUCCESS");
@ -115,6 +129,11 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
assertEquals(expLog.toString(), sLog);
}
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null);
}
/**
* Test the AuditLog format for successful events passing nulls.
*/
@ -131,6 +150,39 @@ private void testSuccessLogNulls(boolean checkIP) {
assertEquals(expLog.toString(), sLog);
}
private void testSuccessLogFormatHelperWithIP(boolean checkIP,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, InetAddress ip) {
testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, ip);
}
/**
* Tests the SuccessLog with two IP addresses.
*
* @param checkIP
* @param appId
* @param attemptId
* @param containerId
*/
private void testSuccessLogFormatHelperWithIP(boolean checkIP,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId) {
testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID,
InetAddress.getLoopbackAddress());
byte[] ipAddr = new byte[] { 100, 10, 10, 1 };
InetAddress addr = null;
try {
addr = InetAddress.getByAddress(ipAddr);
} catch (UnknownHostException uhe) {
// should not happen as long as IP address format
// stays the same
Assert.fail("Check ip address being constructed");
}
testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID,
addr);
}
/**
* Test the AuditLog format for successful events with the various
* parameters.
@ -144,6 +196,7 @@ private void testSuccessLogFormat(boolean checkIP) {
testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID);
testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogNulls(checkIP);
}

53
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
View File

@ -38,6 +38,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
@ -548,11 +549,14 @@ public void testAppRecoverPath() throws IOException {
public void testAppNewKill() throws IOException {
LOG.info("--- START: testAppNewKill ---");
UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
"fooTestAppNewKill", new String[] { "foo_group" });
RMApp application = createNewTestApp(null);
// NEW => KILLED event RMAppEventType.KILL
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
"Application killed by user.");
RMAppEvent event = new RMAppKillByClientEvent(
application.getApplicationId(), "Application killed by user.", fooUser,
Server.getRemoteIp());
application.handle(event);
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
@ -601,11 +605,15 @@ public void testAppNewRejectAddToStore() throws IOException {
public void testAppNewSavingKill() throws IOException {
LOG.info("--- START: testAppNewSavingKill ---");
UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
"fooTestAppNewSavingKill", new String[] { "foo_group" });
RMApp application = testCreateAppNewSaving(null);
// NEW_SAVING => KILLED event RMAppEventType.KILL
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
"Application killed by user.");
RMAppEvent event = new RMAppKillByClientEvent(
application.getApplicationId(), "Application killed by user.", fooUser,
Server.getRemoteIp());
application.handle(event);
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
@ -651,11 +659,13 @@ public void testAppSubmittedRejected() throws IOException {
@Test
public void testAppSubmittedKill() throws IOException, InterruptedException {
LOG.info("--- START: testAppSubmittedKill---");
UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
"fooTestAppSubmittedKill", new String[] { "foo_group" });
RMApp application = testCreateAppSubmittedNoRecovery(null);
// SUBMITTED => KILLED event RMAppEventType.KILL
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
"Application killed by user.");
RMAppEvent event = new RMAppKillByClientEvent(
application.getApplicationId(), "Application killed by user.", fooUser,
Server.getRemoteIp());
application.handle(event);
rmDispatcher.await();
sendAppUpdateSavedEvent(application);
@ -703,11 +713,13 @@ public void testAppAcceptedFailed() throws IOException {
@Test
public void testAppAcceptedKill() throws IOException, InterruptedException {
LOG.info("--- START: testAppAcceptedKill ---");
UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
"fooTestAppAcceptedKill", new String[] { "foo_group" });
RMApp application = testCreateAppAccepted(null);
// ACCEPTED => KILLED event RMAppEventType.KILL
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
"Application killed by user.");
RMAppEvent event = new RMAppKillByClientEvent(
application.getApplicationId(), "Application killed by user.", fooUser,
Server.getRemoteIp());
application.handle(event);
rmDispatcher.await();
@ -750,12 +762,14 @@ public void testAppAcceptedAttemptKilled() throws IOException,
@Test
public void testAppRunningKill() throws IOException {
LOG.info("--- START: testAppRunningKill ---");
UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
"fooTestAppRunningKill", new String[] { "foo_group" });
RMApp application = testCreateAppRunning(null);
// RUNNING => KILLED event RMAppEventType.KILL
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
"Application killed by user.");
RMAppEvent event = new RMAppKillByClientEvent(
application.getApplicationId(), "Application killed by user.", fooUser,
Server.getRemoteIp());
application.handle(event);
rmDispatcher.await();
@ -915,13 +929,14 @@ public void testAppFailedFailed() throws IOException {
@Test (timeout = 30000)
public void testAppKilledKilled() throws IOException {
LOG.info("--- START: testAppKilledKilled ---");
UserGroupInformation fooUser = UserGroupInformation.createUserForTesting(
"fooTestAppKilledKill", new String[] { "foo_group" });
RMApp application = testCreateAppRunning(null);
// RUNNING => KILLED event RMAppEventType.KILL
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.KILL,
"Application killed by user.");
RMAppEvent event = new RMAppKillByClientEvent(
application.getApplicationId(), "Application killed by user.", fooUser,
Server.getRemoteIp());
application.handle(event);
rmDispatcher.await();
sendAttemptUpdateSavedEvent(application);