diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java index 7b527071c3d..c927a66031b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java @@ -140,6 +140,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants; +import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan; import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation; @@ -930,6 +931,8 @@ public void remove() { callerUGI.getUserName(), allowAccess)); } + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_APPLICATIONS_REQUEST, "ClientRMService"); GetApplicationsResponse response = recordFactory.newRecordInstance(GetApplicationsResponse.class); response.setApplicationList(reports); @@ -980,6 +983,13 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) GetQueueInfoResponse response = recordFactory.newRecordInstance(GetQueueInfoResponse.class); + RMAuditLogger.ArgsBuilder arguments = new RMAuditLogger.ArgsBuilder() + .append(Keys.QUEUENAME, request.getQueueName()) + .append(Keys.INCLUDEAPPS, + String.valueOf(request.getIncludeApplications())) + .append(Keys.INCLUDECHILDQUEUES, + String.valueOf(request.getIncludeChildQueues())) + .append(Keys.RECURSIVE, String.valueOf(request.getRecursive())); try { QueueInfo queueInfo = scheduler.getQueueInfo(request.getQueueName(), @@ -1006,8 +1016,14 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request) } queueInfo.setApplications(appReports); response.setQueueInfo(queueInfo); + RMAuditLogger.logSuccess(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, + "ClientRMService", arguments); } catch (IOException ioe) { LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe); + RMAuditLogger.logFailure(callerUGI.getUserName(), + AuditConstants.GET_QUEUE_INFO_REQUEST, "UNKNOWN", "ClientRMService", + ioe.getMessage(), arguments); } return response; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java index 9c69504f27b..07634c7bc7c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java @@ -38,7 +38,8 @@ public class RMAuditLogger { static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS, DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, - CALLERCONTEXT, CALLERSIGNATURE} + CALLERCONTEXT, CALLERSIGNATURE, QUEUENAME, + INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE} public static class AuditConstants { static final String SUCCESS = "SUCCESS"; @@ -53,6 +54,10 @@ public static class AuditConstants { public static final String GET_APP_STATE = "Get Application State"; public static final String GET_APP_PRIORITY = "Get Application Priority"; public static final String GET_APP_QUEUE = "Get Application Queue"; + public static final String GET_QUEUE_INFO_REQUEST = + "Get Queue Info Request"; + public static final String GET_APPLICATIONS_REQUEST = + "Get Applications Request"; public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded"; public static final String FINISH_FAILED_APP = "Application Finished - Failed"; public static final String FINISH_KILLED_APP = "Application Finished - Killed"; @@ -88,12 +93,11 @@ static String createSuccessLog(String user, String operation, String target, } /** - * A helper api for creating an audit log for a successful event. + * A helper function for creating the common portion of a successful + * log message. */ - static String createSuccessLog(String user, String operation, String target, - ApplicationId appId, ApplicationAttemptId attemptId, - ContainerId containerId, CallerContext callerContext, - InetAddress ip) { + private static StringBuilder createStringBuilderForSuccessEvent(String user, + String operation, String target, InetAddress ip) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); if (ip != null) { @@ -102,6 +106,18 @@ static String createSuccessLog(String user, String operation, String target, add(Keys.OPERATION, operation, b); add(Keys.TARGET, target ,b); add(Keys.RESULT, AuditConstants.SUCCESS, b); + return b; + } + + /** + * A helper api for creating an audit log for a successful event. + */ + static String createSuccessLog(String user, String operation, String target, + ApplicationId appId, ApplicationAttemptId attemptId, + ContainerId containerId, CallerContext callerContext, + InetAddress ip) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target, ip); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -138,6 +154,20 @@ private static void appendCallerContext(StringBuilder sb, CallerContext callerCo } } + /** + * A general helper api for creating an audit log for a successful event. + */ + @SuppressWarnings("rawtypes") + static String createSuccessLog(String user, String operation, String target, + InetAddress ip, ArgsBuilder args) { + StringBuilder b = + createStringBuilderForSuccessEvent(user, operation, target, ip); + if(args != null) { + add(args, b); + } + return b.toString(); + } + /** * Create a readable and parseable audit log string for a successful event. * @@ -159,12 +189,53 @@ public static void logSuccess(String user, String operation, String target, } } + /** + * Create a general readable and parseable audit log string for a successful + * event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param args The ArgsBuilder arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + *
+ * This method will attempt to retrieve the remote IP + */ + public static void logSuccess(String user, String operation, String target, + ArgsBuilder args) { + logSuccess(user, operation, target, Server.getRemoteIp(), args); + } + + /** + * Create a general readable and parseable audit log string for a successful + * event. + * + * @param user User who made the service request to the ResourceManager. + * @param operation Operation requested by the user. + * @param target The target on which the operation is being performed. + * @param ip The ip address of the caller. + * @param args The ArgsBuilder arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logSuccess(String user, String operation, String target, + InetAddress ip, ArgsBuilder args) { + if (LOG.isInfoEnabled()) { + LOG.info(createSuccessLog(user, operation, target, ip, args)); + } + } + /** * Create a readable and parseable audit log string for a successful event. * * @param user User who made the service request to the ResourceManager. * @param operation Operation requested by the user. - * @param target The target on which the operation is being performed. + * @param target The target on which the operation is being performed. * @param appId Application Id in which operation was performed. * @param attemptId Application Attempt Id in which operation was performed. * @@ -252,10 +323,8 @@ public static void logSuccess(String user, String operation, String target) { } } - static String createFailureLog(String user, String operation, String perm, - String target, String description, ApplicationId appId, - ApplicationAttemptId attemptId, ContainerId containerId, - CallerContext callerContext) { + private static StringBuilder createStringBuilderForFailureLog(String user, + String operation, String target, String description, String perm) { StringBuilder b = new StringBuilder(); start(Keys.USER, user, b); addRemoteIP(b); @@ -264,6 +333,18 @@ static String createFailureLog(String user, String operation, String perm, add(Keys.RESULT, AuditConstants.FAILURE, b); add(Keys.DESCRIPTION, description, b); add(Keys.PERMISSIONS, perm, b); + return b; + } + + /** + * A helper api for creating an audit log for a failure event. + */ + static String createFailureLog(String user, String operation, String perm, + String target, String description, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + CallerContext callerContext) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); if (appId != null) { add(Keys.APPID, appId.toString(), b); } @@ -287,6 +368,20 @@ static String createFailureLog(String user, String operation, String perm, attemptId, containerId, null); } + /** + * A helper api for creating an audit log for a failure event. + */ + @SuppressWarnings("rawtypes") + static String createFailureLog(String user, String operation, String perm, + String target, String description, ArgsBuilder args) { + StringBuilder b = createStringBuilderForFailureLog(user, + operation, target, description, perm); + if(args != null) { + add(args, b); + } + return b.toString(); + } + /** * Create a readable and parseable audit log string for a failed event. * @@ -391,7 +486,29 @@ public static void logFailure(String user, String operation, String perm, } /** - * A helper api to add remote IP address + * Create a readable and parseable audit log string for a failed event. + * + * @param user User who made the service request. + * @param operation Operation requested by the user. + * @param perm Target permissions. + * @param target The target on which the operation is being performed. + * @param description The failure description + * @param args The arguments for the operation request. + * + *

+ * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter + * and hence the value fields should not contains tabs ('\t'). + */ + public static void logFailure(String user, String operation, String perm, + String target, String description, ArgsBuilder args) { + if (LOG.isWarnEnabled()) { + LOG.warn(createFailureLog(user, operation, perm, target, description, + args)); + } + } + + /** + * A helper api to add remote IP address. */ static void addRemoteIP(StringBuilder b) { InetAddress ip = Server.getRemoteIp(); @@ -417,4 +534,35 @@ static void add(Keys key, String value, StringBuilder b) { b.append(AuditConstants.PAIR_SEPARATOR).append(key.name()) .append(AuditConstants.KEY_VAL_SEPARATOR).append(value); } + + /** + * Appends the key-val pair to the passed builder in the following format + * key=value + */ + static void add(ArgsBuilder args, StringBuilder b) { + b.append(AuditConstants.PAIR_SEPARATOR).append(args.getArgs()); + } + + /** + * Builder to create and pass a list of arbitrary key value pairs for logging. + */ + public static class ArgsBuilder { + private StringBuilder b; + + public ArgsBuilder() { + b = new StringBuilder(); + } + + public ArgsBuilder append(Keys key, String value) { + if (b.length() != 0) { + b.append(AuditConstants.PAIR_SEPARATOR); + } + b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value); + return this; + } + + public StringBuilder getArgs() { + return b; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java index 4d9eaded3af..7ce7a496686 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java @@ -59,6 +59,7 @@ public class TestRMAuditLogger { private static final String TARGET = "tgt"; private static final String PERM = "admin group"; private static final String DESC = "description of an audit log"; + private static final String QUEUE = "root"; private static final ApplicationId APPID = mock(ApplicationId.class); private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class); private static final ContainerId CONTAINERID = mock(ContainerId.class); @@ -119,14 +120,22 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext, InetAddress remoteIp) { + testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId, + callerContext, remoteIp, null); + } + private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + CallerContext callerContext, InetAddress remoteIp, + RMAuditLogger.ArgsBuilder args) { String sLog; - if (checkIP) { - sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, callerContext, remoteIp); + InetAddress tmpIp = checkIP ? remoteIp : null; + if (args != null) { + sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, + tmpIp, args); } else { sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId, - attemptId, containerId, callerContext, null); + attemptId, containerId, callerContext, tmpIp); } StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); @@ -158,6 +167,10 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId, expLog.append("\tCALLERSIGNATURE=signature"); } } + if (args != null) { + expLog.append("\tQUEUENAME=root"); + expLog.append("\tRECURSIVE=true"); + } assertEquals(expLog.toString(), sLog); } @@ -234,6 +247,10 @@ private void testSuccessLogFormat(boolean checkIP) { testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) .build()); + RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder() + .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); + testSuccessLogFormatHelper(checkIP, null, null, null, null, + Server.getRemoteIp(), args); testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID); testSuccessLogNulls(checkIP); } @@ -249,9 +266,18 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, ApplicationAttemptId attemptId, ContainerId containerId, CallerContext callerContext) { - String fLog = + testFailureLogFormatHelper(checkIP, appId, attemptId, containerId, + callerContext, null); + } + + private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, + ApplicationAttemptId attemptId, ContainerId containerId, + CallerContext callerContext, RMAuditLogger.ArgsBuilder args) { + String fLog = args == null ? RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, - appId, attemptId, containerId, callerContext); + appId, attemptId, containerId, callerContext) : + RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC, + args); StringBuilder expLog = new StringBuilder(); expLog.append("USER=test\t"); if (checkIP) { @@ -278,6 +304,10 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId, expLog.append("\tCALLERSIGNATURE=signature"); } } + if (args != null) { + expLog.append("\tQUEUENAME=root"); + expLog.append("\tRECURSIVE=true"); + } assertEquals(expLog.toString(), fLog); } @@ -304,6 +334,9 @@ private void testFailureLogFormat(boolean checkIP) { testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID, new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE) .build()); + RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder() + .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true"); + testFailureLogFormatHelper(checkIP, null, null, null, null, args); } /**