diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index d3a37b1f54f..14a338d928d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -1626,7 +1626,7 @@ public class ClientRMService extends AbstractService implements if (application == null) { RMAuditLogger.logFailure(callerUGI.getUserName(), AuditConstants.SIGNAL_CONTAINER, "UNKNOWN", "ClientRMService", - "Trying to signal an absent container", applicationId, containerId); + "Trying to signal an absent container", applicationId, containerId, null); throw RPCUtil .getRemoteException("Trying to signal an absent container " + containerId); @@ -1650,11 +1650,11 @@ public class ClientRMService extends AbstractService implements request)); RMAuditLogger.logSuccess(callerUGI.getShortUserName(), AuditConstants.SIGNAL_CONTAINER, "ClientRMService", applicationId, - containerId); + containerId, null); } else { RMAuditLogger.logFailure(callerUGI.getUserName(), AuditConstants.SIGNAL_CONTAINER, "UNKNOWN", "ClientRMService", - "Trying to signal an absent container", applicationId, containerId); + "Trying to signal an absent container", applicationId, containerId, null); throw RPCUtil .getRemoteException("Trying to signal an absent container " + containerId); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 3b603a4829c..d08cb9eaec7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -27,6 +27,7 @@ import org.apache.hadoop.ipc.Server; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; /** * Manages ResourceManager audit logs. @@ -38,7 +39,7 @@ public class RMAuditLogger { static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, - CALLERCONTEXT, CALLERSIGNATURE} + CALLERCONTEXT, CALLERSIGNATURE, RESOURCE} public static class AuditConstants { static final String SUCCESS = "SUCCESS"; @@ -77,9 +78,9 @@ public class RMAuditLogger { static String createSuccessLog(String user, String operation, String target, ApplicationId appId, ApplicationAttemptId attemptId, - ContainerId containerId) { + ContainerId containerId, Resource resource) { return createSuccessLog(user, operation, target, appId, attemptId, - containerId, null); + containerId, resource, null); } /** @@ -87,7 +88,7 @@ public class RMAuditLogger { */ static String createSuccessLog(String user, String operation, String target, ApplicationId appId, ApplicationAttemptId attemptId, - ContainerId containerId, CallerContext callerContext) { + ContainerId containerId, Resource resource, CallerContext callerContext) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); addRemoteIP(b); @@ -103,6 +104,9 @@ public class RMAuditLogger { if (containerId != null) { add(Keys.CONTAINERID, containerId.toString(), b); } + if (resource != null) { + add(Keys.RESOURCE, resource.toString(), b); + } appendCallerContext(b, callerContext); return b.toString(); } @@ -138,16 +142,17 @@ public class RMAuditLogger { * @param target The target on which the operation is being performed. * @param appId Application Id in which operation was performed. * @param containerId Container Id in which operation was performed. + * @param resource Resource associated with container. * *

* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter * and hence the value fields should not contains tabs ('\t'). */ public static void logSuccess(String user, String operation, String target, - ApplicationId appId, ContainerId containerId) { + ApplicationId appId, ContainerId containerId, Resource resource) { if (LOG.isInfoEnabled()) { LOG.info(createSuccessLog(user, operation, target, appId, null, - containerId)); + containerId, resource)); } } @@ -168,7 +173,7 @@ public class RMAuditLogger { ApplicationId appId, ApplicationAttemptId attemptId) { if (LOG.isInfoEnabled()) { LOG.info(createSuccessLog(user, operation, target, appId, attemptId, - null)); + null, null)); } } @@ -176,7 +181,7 @@ public class RMAuditLogger { ApplicationId appId, CallerContext callerContext) { if (LOG.isInfoEnabled()) { LOG.info(createSuccessLog(user, operation, target, appId, null, null, - callerContext)); + null, callerContext)); } } @@ -196,7 +201,7 @@ public class RMAuditLogger { public static void logSuccess(String user, String operation, String target, ApplicationId appId) { if (LOG.isInfoEnabled()) { - LOG.info(createSuccessLog(user, operation, target, appId, null, null)); + LOG.info(createSuccessLog(user, operation, target, appId, null, null, null)); } } @@ -213,14 +218,14 @@ public class RMAuditLogger { */ public static void logSuccess(String user, String operation, String target) { if (LOG.isInfoEnabled()) { - LOG.info(createSuccessLog(user, operation, target, null, null, null)); + LOG.info(createSuccessLog(user, operation, target, null, null, null, null)); } } static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - CallerContext callerContext) { + Resource resource, CallerContext callerContext) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); addRemoteIP(b); @@ -238,6 +243,9 @@ public class RMAuditLogger { if (containerId != null) { add(Keys.CONTAINERID, containerId.toString(), b); } + if (resource != null) { + add(Keys.RESOURCE, resource.toString(), b); + } appendCallerContext(b, callerContext); return b.toString(); } @@ -247,9 +255,9 @@ public class RMAuditLogger { */ static String createFailureLog(String user, String operation, String perm, String target, String description, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId) { + ApplicationAttemptId attemptId, ContainerId containerId, Resource resource) { return createFailureLog(user, operation, perm, target, description, appId, - attemptId, containerId, null); + attemptId, containerId, resource, null); } /** @@ -263,6 +271,7 @@ public class RMAuditLogger { * failed. * @param appId Application Id in which operation was performed. * @param containerId Container Id in which operation was performed. + * @param resource Resources associated with container. * *

* Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter @@ -270,10 +279,10 @@ public class RMAuditLogger { */ public static void logFailure(String user, String operation, String perm, String target, String description, ApplicationId appId, - ContainerId containerId) { + ContainerId containerId, Resource resource) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, containerId)); + appId, null, containerId, resource)); } } @@ -297,7 +306,7 @@ public class RMAuditLogger { ApplicationAttemptId attemptId) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, attemptId, null)); + appId, attemptId, null, null)); } } @@ -306,7 +315,7 @@ public class RMAuditLogger { CallerContext callerContext) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null, callerContext)); + appId, null, null, null, callerContext)); } } @@ -329,7 +338,7 @@ public class RMAuditLogger { String target, String description, ApplicationId appId) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - appId, null, null)); + appId, null, null, null)); } } @@ -351,7 +360,7 @@ public class RMAuditLogger { String target, String description) { if (LOG.isWarnEnabled()) { LOG.warn(createFailureLog(user, operation, perm, target, description, - null, null, null)); + null, null, null, null)); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java index 7d123012529..0f79a57aad5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java @@ -503,7 +503,7 @@ public abstract class AbstractYarnScheduler "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app " + "or with invalid id.", attempt.getApplicationId(), - containerId); + containerId, null); } attempt.getPendingRelease().clear(); } @@ -554,7 +554,7 @@ public abstract class AbstractYarnScheduler AuditConstants.RELEASE_CONTAINER, "Unauthorized access or invalid container", "Scheduler", "Trying to release container not owned by app or with invalid id.", - attempt.getApplicationId(), containerId); + attempt.getApplicationId(), containerId, null); } } completedContainer(rmContainer, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 35329d27f38..f764cac5aee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -165,12 +165,12 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { containersToPreempt.remove(containerId); + Resource containerResource = rmContainer.getContainer().getResource(); RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER, "SchedulerApp", - getApplicationId(), containerId); + getApplicationId(), containerId, containerResource); // Update usage metrics - Resource containerResource = rmContainer.getContainer().getResource(); queue.getMetrics().releaseResources(getUser(), 1, containerResource); attemptResourceUsage.decUsed(partition, containerResource); @@ -229,7 +229,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt { } RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", - getApplicationId(), containerId); + getApplicationId(), containerId, container.getResource()); return rmContainer; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java index 0190742fe33..95144a1cbb3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java @@ -146,12 +146,12 @@ public class FSAppAttempt extends SchedulerApplicationAttempt // Remove from the list of containers liveContainers.remove(rmContainer.getContainerId()); + Resource containerResource = rmContainer.getContainer().getResource(); RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER, "SchedulerApp", - getApplicationId(), containerId); + getApplicationId(), containerId, containerResource); // Update usage metrics - Resource containerResource = rmContainer.getContainer().getResource(); queue.getMetrics().releaseResources(getUser(), 1, containerResource); this.attemptResourceUsage.decUsed(containerResource); @@ -403,7 +403,7 @@ public class FSAppAttempt extends SchedulerApplicationAttempt } RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER, "SchedulerApp", - getApplicationId(), container.getId()); + getApplicationId(), container.getId(), container.getResource()); return rmContainer; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 66af3f1e75e..acb8e3705d9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -43,6 +43,7 @@ import org.apache.hadoop.net.NetUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys; import org.junit.Assert; import org.junit.Before; @@ -61,6 +62,7 @@ public class TestRMAuditLogger { private static final ApplicationId APPID = mock(ApplicationId.class); private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class); private static final ContainerId CONTAINERID = mock(ContainerId.class); + private static final Resource RESOURCE = mock(Resource.class); private static final String CALLER_CONTEXT = "context"; private static final byte[] CALLER_SIGNATURE = "signature".getBytes(); @@ -69,6 +71,7 @@ public class TestRMAuditLogger { when(APPID.toString()).thenReturn("app_1"); when(ATTEMPTID.toString()).thenReturn("app_attempt_1"); when(CONTAINERID.toString()).thenReturn("container_1"); + when(RESOURCE.toString()).thenReturn(""); } @@ -99,10 +102,11 @@ public class TestRMAuditLogger { expLog.append("\tTARGET=tgt"); assertEquals(expLog.toString(), actLog.toString()); } - + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { - testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null); + testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, null, + null); } /** @@ -110,9 +114,9 @@ public class TestRMAuditLogger { */ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - CallerContext callerContext) { + CallerContext callerContext, Resource resource) { String sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, - appId, attemptId, containerId, callerContext); + appId, attemptId, containerId, resource, callerContext); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); if (checkIP) { @@ -130,6 +134,9 @@ public class TestRMAuditLogger { if (containerId != null) { expLog.append("\tCONTAINERID=container_1"); } + if (resource != null) { + expLog.append("\tRESOURCE="); + } if (callerContext != null) { if (callerContext.getContext() != null) { expLog.append("\tCALLERCONTEXT=context"); @@ -146,7 +153,7 @@ public class TestRMAuditLogger { */ private void testSuccessLogNulls(boolean checkIP) { String sLog = RMAuditLogger.createSuccessLog(null, null, null, null, - null, null); + null, null, null); StringBuilder expLog = new StringBuilder(); expLog.append("USER=null\t"); if (checkIP) { @@ -170,22 +177,22 @@ public class TestRMAuditLogger { testSuccessLogFormatHelper(checkIP, APPID, null, CONTAINERID); testSuccessLogFormatHelper(checkIP, null, ATTEMPTID, CONTAINERID); testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); - testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, null); + testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, null, null); testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(null).setSignature(null).build()); + new CallerContext.Builder(null).setSignature(null).build(), RESOURCE); testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build()); + new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build(), RESOURCE); testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build()); + new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build(), RESOURCE); testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) - .build()); + .build(), RESOURCE); testSuccessLogNulls(checkIP); } private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId) { - testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, null); + testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, null, null); } /** @@ -193,10 +200,10 @@ public class TestRMAuditLogger { */ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, - CallerContext callerContext) { + CallerContext callerContext, Resource resource) { String fLog = RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, callerContext); + appId, attemptId, containerId, resource, callerContext); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); if (checkIP) { @@ -215,6 +222,9 @@ public class TestRMAuditLogger { if (containerId != null) { expLog.append("\tCONTAINERID=container_1"); } + if (resource != null) { + expLog.append("\tRESOURCE="); + } if (callerContext != null) { if (callerContext.getContext() != null) { expLog.append("\tCALLERCONTEXT=context"); @@ -241,14 +251,14 @@ public class TestRMAuditLogger { testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(null).setSignature(null).build()); + new CallerContext.Builder(null).setSignature(null).build(), RESOURCE); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build()); + new CallerContext.Builder(CALLER_CONTEXT).setSignature(null).build(), RESOURCE); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, - new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build()); + new CallerContext.Builder(null).setSignature(CALLER_SIGNATURE).build(), RESOURCE); testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) - .build()); + .build(), RESOURCE); } /**