diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 26471308f30..1fe2237ff20 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -144,6 +144,7 @@ import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation; @@ -921,6 +922,8 @@ public class ClientRMService extends AbstractService implements callerUGI.getUserName(), allowAccess)); } + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_APPLICATIONS_REQUEST, "ClientRMService"); GetApplicationsResponse response = recordFactory.newRecordInstance(GetApplicationsResponse.class); response.setApplicationList(reports); @@ -971,6 +974,13 @@ public class ClientRMService extends AbstractService implements GetQueueInfoResponse response = recordFactory.newRecordInstance(GetQueueInfoResponse.class); + RMAuditLogger.ArgsBuilder arguments = new RMAuditLogger.ArgsBuilder() + .append(Keys.QUEUENAME, request.getQueueName()) + .append(Keys.INCLUDEAPPS, + String.valueOf(request.getIncludeApplications())) + .append(Keys.INCLUDECHILDQUEUES, + String.valueOf(request.getIncludeChildQueues())) + .append(Keys.RECURSIVE, String.valueOf(request.getRecursive())); try { QueueInfo queueInfo = scheduler.getQueueInfo(request.getQueueName(), @@ -997,8 +1007,14 @@ public class ClientRMService extends AbstractService implements } queueInfo.setApplications(appReports); response.setQueueInfo(queueInfo); + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, + "ClientRMService", arguments); } catch (IOException ioe) { LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe); + RMAuditLogger.logFailure(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, "UNKNOWN", "ClientRMService", + ioe.getMessage(), arguments); } return response; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index fc0b2655802..4f191a6d9e1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -39,7 +39,8 @@ public class RMAuditLogger { static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, - CALLERCONTEXT, CALLERSIGNATURE, RESOURCE} + CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUENAME, + INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE} public static class AuditConstants { static final String SUCCESS = "SUCCESS"; @@ -59,6 +60,10 @@ public class RMAuditLogger { = "Get Application Attempt Report"; public static final String GET_CONTAINERS = "Get Containers"; public static final String GET_CONTAINER_REPORT = "Get Container Report"; + public static final String GET_QUEUE_INFO_REQUEST = + "Get Queue Info Request"; + public static final String GET_APPLICATIONS_REQUEST = + "Get Applications Request"; public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; public static final String FINISH_FAILED_APP = "Application Finished - Failed"; public static final String FINISH_KILLED_APP = "Application Finished - Killed"; @@ -97,12 +102,11 @@ public class RMAuditLogger { } /** - * A helper api for creating an audit log for a successful event. + * A helper function for creating the common portion of a successful + * log message. */ - static String createSuccessLog(String user, String operation, String target, - ApplicationId appId, ApplicationAttemptId attemptId, - ContainerId containerId, Resource resource, CallerContext callerContext, - InetAddress ip) { + private static StringBuilder createStringBuilderForSuccessEvent(String user, + String operation, String target, InetAddress ip) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); if (ip != null) { @@ -111,6 +115,18 @@ public class RMAuditLogger { add(Keys.OPERATION, operation, b); add(Keys.TARGET, target ,b); add(Keys.RESULT, AuditConstants.SUCCESS, b); + return b; + } + + /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId, Resource resource, CallerContext callerContext, + InetAddress ip) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target, ip); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -150,6 +166,20 @@ public class RMAuditLogger { } } + /** + * A general helper api for creating an audit log for a successful event. + */ + @SuppressWarnings("rawtypes") + static String createSuccessLog(String user, String operation, String target, + InetAddress ip, ArgsBuilder args) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target, ip); + if(args != null) { + add(args, b); + } + return b.toString(); + } + /** * Create a readable and parseable audit log string for a successful event. * @@ -172,12 +202,53 @@ public class RMAuditLogger { } } + /** + * Create a general readable and parseable audit log string for a successful + * event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param args The ArgsBuilder arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + *
+ * This method will attempt to retrieve the remote IP + */ + public static void logSuccess(String user, String operation, String target, + ArgsBuilder args) { + logSuccess(user, operation, target, Server.getRemoteIp(), args); + } + + /** + * Create a general readable and parseable audit log string for a successful + * event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param ip The ip address of the caller. + * @param args The ArgsBuilder arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + InetAddress ip, ArgsBuilder args) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, ip, args)); + } + } + /** * Create a readable and parseable audit log string for a successful event. * * @param user User who made the service request to the ResourceManager. * @param operation Operation requested by the user. - * @param target The target on which the operation is being performed. + * @param target The target on which the operation is being performed. * @param appId Application Id in which operation was performed. * @param attemptId Application Attempt Id in which operation was performed. * @@ -265,10 +336,8 @@ public class RMAuditLogger { } } - static String createFailureLog(String user, String operation, String perm, - String target, String description, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId, - Resource resource, CallerContext callerContext) { + private static StringBuilder createStringBuilderForFailureLog(String user, + String operation, String target, String description, String perm) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); addRemoteIP(b); @@ -277,6 +346,18 @@ public class RMAuditLogger { add(Keys.RESULT, AuditConstants.FAILURE, b); add(Keys.DESCRIPTION, description, b); add(Keys.PERMISSIONS, perm, b); + return b; + } + + /** + * A helper api for creating an audit log for a failure event. + */ + static String createFailureLog(String user, String operation, String perm, + String target, String description, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + Resource resource, CallerContext callerContext) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -303,6 +384,20 @@ public class RMAuditLogger { attemptId, containerId, resource, null); } + /** + * A helper api for creating an audit log for a failure event. + */ + @SuppressWarnings("rawtypes") + static String createFailureLog(String user, String operation, String perm, + String target, String description, ArgsBuilder args) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); + if(args != null) { + add(args, b); + } + return b.toString(); + } + /** * Create a readable and parseable audit log string for a failed event. * @@ -408,7 +503,29 @@ public class RMAuditLogger { } /** - * A helper api to add remote IP address + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description The failure description + * @param args The arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ArgsBuilder args) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + args)); + } + } + + /** + * A helper api to add remote IP address. */ static void addRemoteIP(StringBuilder b) { InetAddress ip = Server.getRemoteIp(); @@ -434,4 +551,35 @@ public class RMAuditLogger { b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value + */ + static void add(ArgsBuilder args, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(args.getArgs()); + } + + /** + * Builder to create and pass a list of arbitrary key value pairs for logging. + */ + public static class ArgsBuilder { + private StringBuilder b; + + public ArgsBuilder() { + b = new StringBuilder(); + } + + public ArgsBuilder append(Keys key, String value) { + if (b.length() != 0) { + b.append(AuditConstants.PAIR_SEPARATOR); + } + b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + return this; + } + + public StringBuilder getArgs() { + return b; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 3311f92834f..d5f1e35c983 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -60,6 +60,7 @@ public class TestRMAuditLogger { private static final String TARGET = "tgt"; private static final String PERM = "admin group"; private static final String DESC = "description of an audit log"; + private static final String QUEUE = "root"; private static final ApplicationId APPID = mock(ApplicationId.class); private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class); private static final ContainerId CONTAINERID = mock(ContainerId.class); @@ -123,14 +124,22 @@ public class TestRMAuditLogger { private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource, InetAddress remoteIp) { + testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, + callerContext, resource, remoteIp, null); + } + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + CallerContext callerContext, Resource resource, InetAddress remoteIp, + RMAuditLogger.ArgsBuilder args) { String sLog; - if (checkIP) { - sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, resource, callerContext, remoteIp); + InetAddress tmpIp = checkIP ? remoteIp : null; + if (args != null) { + sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, + tmpIp, args); } else { sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, resource, callerContext, null); + attemptId, containerId, resource, callerContext, tmpIp); } StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); @@ -165,6 +174,10 @@ public class TestRMAuditLogger { expLog.append("\tCALLERSIGNATURE=signature"); } } + if (args != null) { + expLog.append("\tQUEUENAME=root"); + expLog.append("\tRECURSIVE=true"); + } assertEquals(expLog.toString(), sLog); } @@ -241,6 +254,10 @@ public class TestRMAuditLogger { testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) .build(), RESOURCE); + RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder() + .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); + testSuccessLogFormatHelper(checkIP, null, null, null, null, null, + Server.getRemoteIp(), args); testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID); testSuccessLogNulls(checkIP); } @@ -256,9 +273,19 @@ public class TestRMAuditLogger { private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, Resource resource) { - String fLog = + testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, + callerContext, resource, null); + } + + private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + CallerContext callerContext, Resource resource, + RMAuditLogger.ArgsBuilder args) { + String fLog = args == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, resource, callerContext); + appId, attemptId, containerId, resource, callerContext) : + RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, + args); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); if (checkIP) { @@ -288,6 +315,10 @@ public class TestRMAuditLogger { expLog.append("\tCALLERSIGNATURE=signature"); } } + if (args != null) { + expLog.append("\tQUEUENAME=root"); + expLog.append("\tRECURSIVE=true"); + } assertEquals(expLog.toString(), fLog); } @@ -314,6 +345,10 @@ public class TestRMAuditLogger { testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) .build(), RESOURCE); + RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder() + .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); + testFailureLogFormatHelper(checkIP, null, null, null, null, null, + args); } /**