Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

YARN-5003. Add container resource to RM audit log. Contributed by Nathan Roberts 

(cherry picked from commit ed54f5f1ff)
This commit is contained in:
Jason Lowe 2016-05-03 20:03:41 +00:00
parent d8fcf02069
commit baac4e7db1
6 changed files with 66 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
View File

@ -1626,7 +1626,7 @@ public SignalContainerResponse signalContainer(
if (application == null) {
RMAuditLogger.logFailure(callerUGI.getUserName(),
AuditConstants.SIGNAL_CONTAINER, "UNKNOWN", "ClientRMService",
"Trying to signal an absent container", applicationId, containerId);
"Trying to signal an absent container", applicationId, containerId, null);
throw RPCUtil
.getRemoteException("Trying to signal an absent container "
+ containerId);
@ -1650,11 +1650,11 @@ public SignalContainerResponse signalContainer(
request));
RMAuditLogger.logSuccess(callerUGI.getShortUserName(),
AuditConstants.SIGNAL_CONTAINER, "ClientRMService", applicationId,
containerId);
containerId, null);
} else {
RMAuditLogger.logFailure(callerUGI.getUserName(),
AuditConstants.SIGNAL_CONTAINER, "UNKNOWN", "ClientRMService",
"Trying to signal an absent container", applicationId, containerId);
"Trying to signal an absent container", applicationId, containerId, null);
throw RPCUtil
.getRemoteException("Trying to signal an absent container "
+ containerId);

47
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java
View File

@ -27,6 +27,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
/**
* Manages ResourceManager audit logs.
@ -38,7 +39,7 @@ public class RMAuditLogger {
static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID,
CALLERCONTEXT, CALLERSIGNATURE}
CALLERCONTEXT, CALLERSIGNATURE, RESOURCE}
public static class AuditConstants {
static final String SUCCESS = "SUCCESS";
@ -77,9 +78,9 @@ public static class AuditConstants {
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId) {
ContainerId containerId, Resource resource) {
return createSuccessLog(user, operation, target, appId, attemptId,
containerId, null);
containerId, resource, null);
}
/**
@ -87,7 +88,7 @@ static String createSuccessLog(String user, String operation, String target,
*/
static String createSuccessLog(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId,
ContainerId containerId, CallerContext callerContext) {
ContainerId containerId, Resource resource, CallerContext callerContext) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
@ -103,6 +104,9 @@ static String createSuccessLog(String user, String operation, String target,
if (containerId != null) {
add(Keys.CONTAINERID, containerId.toString(), b);
}
if (resource != null) {
add(Keys.RESOURCE, resource.toString(), b);
}
appendCallerContext(b, callerContext);
return b.toString();
}
@ -138,16 +142,17 @@ private static void appendCallerContext(StringBuilder sb, CallerContext callerCo
* @param target The target on which the operation is being performed.
* @param appId Application Id in which operation was performed.
* @param containerId Container Id in which operation was performed.
* @param resource Resource associated with container.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
* and hence the value fields should not contains tabs ('\t').
*/
public static void logSuccess(String user, String operation, String target,
ApplicationId appId, ContainerId containerId) {
ApplicationId appId, ContainerId containerId, Resource resource) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null,
containerId));
containerId, resource));
}
}
@ -168,7 +173,7 @@ public static void logSuccess(String user, String operation, String target,
ApplicationId appId, ApplicationAttemptId attemptId) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, attemptId,
null));
null, null));
}
}
@ -176,7 +181,7 @@ public static void logSuccess(String user, String operation, String target,
ApplicationId appId, CallerContext callerContext) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null, null,
callerContext));
null, callerContext));
}
}
@ -196,7 +201,7 @@ public static void logSuccess(String user, String operation, String target,
public static void logSuccess(String user, String operation, String target,
ApplicationId appId) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, appId, null, null));
LOG.info(createSuccessLog(user, operation, target, appId, null, null, null));
}
}
@ -213,14 +218,14 @@ public static void logSuccess(String user, String operation, String target,
*/
public static void logSuccess(String user, String operation, String target) {
if (LOG.isInfoEnabled()) {
LOG.info(createSuccessLog(user, operation, target, null, null, null));
LOG.info(createSuccessLog(user, operation, target, null, null, null, null));
}
}
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext) {
Resource resource, CallerContext callerContext) {
StringBuilder b = new StringBuilder();
start(Keys.USER, user, b);
addRemoteIP(b);
@ -238,6 +243,9 @@ static String createFailureLog(String user, String operation, String perm,
if (containerId != null) {
add(Keys.CONTAINERID, containerId.toString(), b);
}
if (resource != null) {
add(Keys.RESOURCE, resource.toString(), b);
}
appendCallerContext(b, callerContext);
return b.toString();
}
@ -247,9 +255,9 @@ static String createFailureLog(String user, String operation, String perm,
*/
static String createFailureLog(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) {
return createFailureLog(user, operation, perm, target, description, appId,
attemptId, containerId, null);
attemptId, containerId, resource, null);
}
/**
@ -263,6 +271,7 @@ static String createFailureLog(String user, String operation, String perm,
* failed.
* @param appId Application Id in which operation was performed.
* @param containerId Container Id in which operation was performed.
* @param resource Resources associated with container.
*
* <br><br>
* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
@ -270,10 +279,10 @@ static String createFailureLog(String user, String operation, String perm,
*/
public static void logFailure(String user, String operation, String perm,
String target, String description, ApplicationId appId,
ContainerId containerId) {
ContainerId containerId, Resource resource) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
appId, null, containerId));
appId, null, containerId, resource));
}
}
@ -297,7 +306,7 @@ public static void logFailure(String user, String operation, String perm,
ApplicationAttemptId attemptId) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
appId, attemptId, null));
appId, attemptId, null, null));
}
}
@ -306,7 +315,7 @@ public static void logFailure(String user, String operation, String perm,
CallerContext callerContext) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
appId, null, null, callerContext));
appId, null, null, null, callerContext));
}
}
@ -329,7 +338,7 @@ public static void logFailure(String user, String operation, String perm,
String target, String description, ApplicationId appId) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
appId, null, null));
appId, null, null, null));
}
}
@ -351,7 +360,7 @@ public static void logFailure(String user, String operation, String perm,
String target, String description) {
if (LOG.isWarnEnabled()) {
LOG.warn(createFailureLog(user, operation, perm, target, description,
null, null, null));
null, null, null, null));
}
}

4
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
View File

@ -503,7 +503,7 @@ public void clearPendingContainerCache() {
"Unauthorized access or invalid container", "Scheduler",
"Trying to release container not owned by app "
+ "or with invalid id.", attempt.getApplicationId(),
containerId);
containerId, null);
}
attempt.getPendingRelease().clear();
}
@ -554,7 +554,7 @@ protected void releaseContainers(List<ContainerId> containers,
AuditConstants.RELEASE_CONTAINER,
"Unauthorized access or invalid container", "Scheduler",
"Trying to release container not owned by app or with invalid id.",
attempt.getApplicationId(), containerId);
attempt.getApplicationId(), containerId, null);
}
}
completedContainer(rmContainer,

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
View File

@ -165,12 +165,12 @@ public synchronized boolean containerCompleted(RMContainer rmContainer,
containersToPreempt.remove(containerId);
Resource containerResource = rmContainer.getContainer().getResource();
RMAuditLogger.logSuccess(getUser(),
AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
getApplicationId(), containerId);
getApplicationId(), containerId, containerResource);
// Update usage metrics
Resource containerResource = rmContainer.getContainer().getResource();
queue.getMetrics().releaseResources(getUser(), 1, containerResource);
attemptResourceUsage.decUsed(partition, containerResource);
@ -229,7 +229,7 @@ public synchronized RMContainer allocate(NodeType type, FiCaSchedulerNode node,
}
RMAuditLogger.logSuccess(getUser(),
AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
getApplicationId(), containerId);
getApplicationId(), containerId, container.getResource());
return rmContainer;
}

6
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
View File

@ -146,12 +146,12 @@ synchronized public void containerCompleted(RMContainer rmContainer,
// Remove from the list of containers
liveContainers.remove(rmContainer.getContainerId());
Resource containerResource = rmContainer.getContainer().getResource();
RMAuditLogger.logSuccess(getUser(),
AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
getApplicationId(), containerId);
getApplicationId(), containerId, containerResource);
// Update usage metrics
Resource containerResource = rmContainer.getContainer().getResource();
queue.getMetrics().releaseResources(getUser(), 1, containerResource);
this.attemptResourceUsage.decUsed(containerResource);
@ -403,7 +403,7 @@ else if (allowed.equals(NodeType.RACK_LOCAL) &&
}
RMAuditLogger.logSuccess(getUser(),
AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
getApplicationId(), container.getId());
getApplicationId(), container.getId(), container.getResource());
return rmContainer;
}

42
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
View File

@ -43,6 +43,7 @@
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
import org.junit.Assert;
import org.junit.Before;
@ -61,6 +62,7 @@ public class TestRMAuditLogger {
private static final ApplicationId APPID = mock(ApplicationId.class);
private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class);
private static final ContainerId CONTAINERID = mock(ContainerId.class);
private static final Resource RESOURCE = mock(Resource.class);
private static final String CALLER_CONTEXT = "context";
private static final byte[] CALLER_SIGNATURE = "signature".getBytes();
@ -69,6 +71,7 @@ public void setUp() throws Exception {
when(APPID.toString()).thenReturn("app_1");
when(ATTEMPTID.toString()).thenReturn("app_attempt_1");
when(CONTAINERID.toString()).thenReturn("container_1");
when(RESOURCE.toString()).thenReturn("<memory:1536, vcores:1>");
}
@ -102,7 +105,8 @@ public void testKeyValLogFormat() throws Exception {
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null);
testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null,
null);
}
/**
@ -110,9 +114,9 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
*/
private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext) {
CallerContext callerContext, Resource resource) {
String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
appId, attemptId, containerId, callerContext);
appId, attemptId, containerId, resource, callerContext);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
@ -130,6 +134,9 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
if (containerId != null) {
expLog.append("\tCONTAINERID=container_1");
}
if (resource != null) {
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
if (callerContext.getContext() != null) {
expLog.append("\tCALLERCONTEXT=context");
@ -146,7 +153,7 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
*/
private void testSuccessLogNulls(boolean checkIP) {
String sLog = RMAuditLogger.createSuccessLog(null, null, null, null,
null, null);
null, null, null);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=null\t");
if (checkIP) {
@ -170,22 +177,22 @@ private void testSuccessLogFormat(boolean checkIP) {
testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID);
testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, null);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, null, null);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(null).setSignature(null).build());
new CallerContext.Builder(null).setSignature(null).build(), RESOURCE);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build());
new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build(), RESOURCE);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build());
new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build(), RESOURCE);
testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
.build());
.build(), RESOURCE);
testSuccessLogNulls(checkIP);
}
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId) {
testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, null);
testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, null, null);
}
/**
@ -193,10 +200,10 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
*/
private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
ApplicationAttemptId attemptId, ContainerId containerId,
CallerContext callerContext) {
CallerContext callerContext, Resource resource) {
String fLog =
RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
appId, attemptId, containerId, callerContext);
appId, attemptId, containerId, resource, callerContext);
StringBuilder expLog = new StringBuilder();
expLog.append("USER=test\t");
if (checkIP) {
@ -215,6 +222,9 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
if (containerId != null) {
expLog.append("\tCONTAINERID=container_1");
}
if (resource != null) {
expLog.append("\tRESOURCE=<memory:1536, vcores:1>");
}
if (callerContext != null) {
if (callerContext.getContext() != null) {
expLog.append("\tCALLERCONTEXT=context");
@ -241,14 +251,14 @@ private void testFailureLogFormat(boolean checkIP) {
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID);
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(null).setSignature(null).build());
new CallerContext.Builder(null).setSignature(null).build(), RESOURCE);
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build());
new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build(), RESOURCE);
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build());
new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build(), RESOURCE);
testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
.build());
.build(), RESOURCE);
}
/**