YARN-4163: Audit getQueueInfo and getApplications calls

(cherry picked from commit 921338cd86e7215b0c4b1efdf2daf9449fb12c7b)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java

@@ -144,6 +144,7 @@
 import org.apache.hadoop.yarn.ipc.YarnRPC;
 import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.Keys;
 import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
 import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation;
@@ -921,6 +922,8 @@ public void remove() {
           callerUGI.getUserName(), allowAccess));
     }
 
+    RMAuditLogger.logSuccess(callerUGI.getUserName(),
+        AuditConstants.GET_APPLICATIONS_REQUEST, "ClientRMService");
     GetApplicationsResponse response =
       recordFactory.newRecordInstance(GetApplicationsResponse.class);
     response.setApplicationList(reports);
@@ -971,6 +974,13 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
 
     GetQueueInfoResponse response =
       recordFactory.newRecordInstance(GetQueueInfoResponse.class);
+    RMAuditLogger.ArgsBuilder arguments = new RMAuditLogger.ArgsBuilder()
+        .append(Keys.QUEUENAME, request.getQueueName())
+        .append(Keys.INCLUDEAPPS,
+            String.valueOf(request.getIncludeApplications()))
+        .append(Keys.INCLUDECHILDQUEUES,
+            String.valueOf(request.getIncludeChildQueues()))
+        .append(Keys.RECURSIVE, String.valueOf(request.getRecursive()));
     try {
       QueueInfo queueInfo = 
         scheduler.getQueueInfo(request.getQueueName(),  
@@ -997,8 +1007,14 @@ public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
       }
       queueInfo.setApplications(appReports);
       response.setQueueInfo(queueInfo);
+      RMAuditLogger.logSuccess(callerUGI.getUserName(),
+          AuditConstants.GET_QUEUE_INFO_REQUEST,
+          "ClientRMService", arguments);
     } catch (IOException ioe) {
       LOG.info("Failed to getQueueInfo for " + request.getQueueName(), ioe);
+      RMAuditLogger.logFailure(callerUGI.getUserName(),
+          AuditConstants.GET_QUEUE_INFO_REQUEST, "UNKNOWN", "ClientRMService",
+          ioe.getMessage(), arguments);
     }
     
     return response;

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAuditLogger.java

@@ -39,7 +39,8 @@ public class RMAuditLogger {
 
   static enum Keys {USER, OPERATION, TARGET, RESULT, IP, PERMISSIONS,
                     DESCRIPTION, APPID, APPATTEMPTID, CONTAINERID, 
-                    CALLERCONTEXT, CALLERSIGNATURE, RESOURCE}
+                    CALLERCONTEXT, CALLERSIGNATURE, RESOURCE, QUEUENAME,
+                    INCLUDEAPPS, INCLUDECHILDQUEUES, RECURSIVE}
 
   public static class AuditConstants {
     static final String SUCCESS = "SUCCESS";
@@ -59,6 +60,10 @@ public static class AuditConstants {
         = "Get Application Attempt Report";
     public static final String GET_CONTAINERS = "Get Containers";
     public static final String GET_CONTAINER_REPORT = "Get Container Report";
+    public static final String GET_QUEUE_INFO_REQUEST =
+        "Get Queue Info Request";
+    public static final String GET_APPLICATIONS_REQUEST =
+        "Get Applications Request";
     public static final String FINISH_SUCCESS_APP = "Application Finished - Succeeded";
     public static final String FINISH_FAILED_APP = "Application Finished - Failed";
     public static final String FINISH_KILLED_APP = "Application Finished - Killed";
@@ -97,12 +102,11 @@ static String createSuccessLog(String user, String operation, String target,
   }
 
   /**
-   * A helper api for creating an audit log for a successful event.
+   * A helper function for creating the common portion of a successful
+   * log message.
    */
-  static String createSuccessLog(String user, String operation, String target,
+  private static StringBuilder createStringBuilderForSuccessEvent(String user,
-      ApplicationId appId, ApplicationAttemptId attemptId,
+      String operation, String target, InetAddress ip) {
-      ContainerId containerId, Resource resource, CallerContext callerContext,
-      InetAddress ip) {
     StringBuilder b = new StringBuilder();
     start(Keys.USER, user, b);
     if (ip != null) {
@@ -111,6 +115,18 @@ static String createSuccessLog(String user, String operation, String target,
     add(Keys.OPERATION, operation, b);
     add(Keys.TARGET, target ,b);
     add(Keys.RESULT, AuditConstants.SUCCESS, b);
+    return b;
+  }
+
+  /**
+   * A helper api for creating an audit log for a successful event.
+   */
+  static String createSuccessLog(String user, String operation, String target,
+      ApplicationId appId, ApplicationAttemptId attemptId,
+      ContainerId containerId, Resource resource, CallerContext callerContext,
+      InetAddress ip) {
+    StringBuilder b =
+        createStringBuilderForSuccessEvent(user, operation, target, ip);
     if (appId != null) {
       add(Keys.APPID, appId.toString(), b);
     }
@@ -150,6 +166,20 @@ private static void appendCallerContext(StringBuilder sb, CallerContext callerCo
     }
   }
 
+  /**
+   * A general helper api for creating an audit log for a successful event.
+   */
+  @SuppressWarnings("rawtypes")
+  static String createSuccessLog(String user, String operation, String target,
+      InetAddress ip, ArgsBuilder args) {
+    StringBuilder b =
+        createStringBuilderForSuccessEvent(user, operation, target, ip);
+    if(args != null) {
+      add(args, b);
+    }
+    return b.toString();
+  }
+
   /**
    * Create a readable and parseable audit log string for a successful event.
    *
@@ -172,12 +202,53 @@ public static void logSuccess(String user, String operation, String target,
     }
   }
 
+  /**
+   * Create a general readable and parseable audit log string for a successful
+   * event.
+   *
+   * @param user User who made the service request to the ResourceManager.
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed.
+   * @param args The ArgsBuilder arguments for the operation request.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   * <br>
+   * This method will attempt to retrieve the remote IP
+   */
+  public static void logSuccess(String user, String operation, String target,
+      ArgsBuilder args) {
+    logSuccess(user, operation, target, Server.getRemoteIp(), args);
+  }
+
+  /**
+   * Create a general readable and parseable audit log string for a successful
+   * event.
+   *
+   * @param user User who made the service request to the ResourceManager.
+   * @param operation Operation requested by the user.
+   * @param target The target on which the operation is being performed.
+   * @param ip The ip address of the caller.
+   * @param args The ArgsBuilder arguments for the operation request.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logSuccess(String user, String operation, String target,
+      InetAddress ip, ArgsBuilder args) {
+    if (LOG.isInfoEnabled()) {
+      LOG.info(createSuccessLog(user, operation, target, ip, args));
+    }
+  }
+
   /**
    * Create a readable and parseable audit log string for a successful event.
    *
    * @param user User who made the service request to the ResourceManager.
    * @param operation Operation requested by the user.
-   * @param target The target on which the operation is being performed. 
+   * @param target The target on which the operation is being performed.
    * @param appId Application Id in which operation was performed.
    * @param attemptId Application Attempt Id in which operation was performed.
    *
@@ -265,10 +336,8 @@ public static void logSuccess(String user, String operation, String target) {
     }
   }
   
-  static String createFailureLog(String user, String operation, String perm,
+  private static StringBuilder createStringBuilderForFailureLog(String user,
-      String target, String description, ApplicationId appId,
+      String operation, String target, String description, String perm) {
-      ApplicationAttemptId attemptId, ContainerId containerId,
-      Resource resource, CallerContext callerContext) {
     StringBuilder b = new StringBuilder();
     start(Keys.USER, user, b);
     addRemoteIP(b);
@@ -277,6 +346,18 @@ static String createFailureLog(String user, String operation, String perm,
     add(Keys.RESULT, AuditConstants.FAILURE, b);
     add(Keys.DESCRIPTION, description, b);
     add(Keys.PERMISSIONS, perm, b);
+    return b;
+  }
+
+  /**
+   * A helper api for creating an audit log for a failure event.
+   */
+  static String createFailureLog(String user, String operation, String perm,
+      String target, String description, ApplicationId appId,
+      ApplicationAttemptId attemptId, ContainerId containerId,
+      Resource resource, CallerContext callerContext) {
+    StringBuilder b = createStringBuilderForFailureLog(user,
+        operation, target, description, perm);
     if (appId != null) {
       add(Keys.APPID, appId.toString(), b);
     }
@@ -303,6 +384,20 @@ static String createFailureLog(String user, String operation, String perm,
         attemptId, containerId, resource, null);
   }
 
+  /**
+   * A helper api for creating an audit log for a failure event.
+   */
+  @SuppressWarnings("rawtypes")
+  static String createFailureLog(String user, String operation, String perm,
+      String target, String description, ArgsBuilder args) {
+    StringBuilder b = createStringBuilderForFailureLog(user,
+        operation, target, description, perm);
+    if(args != null) {
+      add(args, b);
+    }
+    return b.toString();
+  }
+
   /**
    * Create a readable and parseable audit log string for a failed event.
    *
@@ -408,7 +503,29 @@ public static void logFailure(String user, String operation, String perm,
   }
 
   /**
-   * A helper api to add remote IP address
+   * Create a readable and parseable audit log string for a failed event.
+   *
+   * @param user User who made the service request.
+   * @param operation Operation requested by the user.
+   * @param perm Target permissions.
+   * @param target The target on which the operation is being performed.
+   * @param description The failure description
+   * @param args The arguments for the operation request.
+   *
+   * <br><br>
+   * Note that the {@link RMAuditLogger} uses tabs ('\t') as a key-val delimiter
+   * and hence the value fields should not contains tabs ('\t').
+   */
+  public static void logFailure(String user, String operation, String perm,
+      String target, String description, ArgsBuilder args) {
+    if (LOG.isWarnEnabled()) {
+      LOG.warn(createFailureLog(user, operation, perm, target, description,
+          args));
+    }
+  }
+
+  /**
+   * A helper api to add remote IP address.
    */
   static void addRemoteIP(StringBuilder b) {
     InetAddress ip = Server.getRemoteIp();
@@ -434,4 +551,35 @@ static void add(Keys key, String value, StringBuilder b) {
     b.append(AuditConstants.PAIR_SEPARATOR).append(key.name())
      .append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
   }
+
+  /**
+   * Appends the key-val pair to the passed builder in the following format
+   * <pair-delim>key=value
+   */
+  static void add(ArgsBuilder args, StringBuilder b) {
+    b.append(AuditConstants.PAIR_SEPARATOR).append(args.getArgs());
+  }
+
+  /**
+   * Builder to create and pass a list of arbitrary key value pairs for logging.
+   */
+  public static class ArgsBuilder {
+    private StringBuilder b;
+
+    public ArgsBuilder() {
+      b = new StringBuilder();
+    }
+
+    public ArgsBuilder append(Keys key, String value) {
+      if (b.length() != 0) {
+        b.append(AuditConstants.PAIR_SEPARATOR);
+      }
+      b.append(key.name()).append(AuditConstants.KEY_VAL_SEPARATOR).append(value);
+      return this;
+    }
+
+    public StringBuilder getArgs() {
+      return b;
+    }
+  }
 }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java

@@ -60,6 +60,7 @@ public class TestRMAuditLogger {
   private static final String TARGET = "tgt";
   private static final String PERM = "admin group";
   private static final String DESC = "description of an audit log";
+  private static final String QUEUE = "root";
   private static final ApplicationId APPID = mock(ApplicationId.class);
   private static final ApplicationAttemptId ATTEMPTID = mock(ApplicationAttemptId.class);
   private static final ContainerId CONTAINERID = mock(ContainerId.class);
@@ -123,14 +124,22 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
   private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
       ApplicationAttemptId attemptId, ContainerId containerId,
       CallerContext callerContext, Resource resource, InetAddress remoteIp) {
+    testSuccessLogFormatHelper(checkIP, appId, attemptId, containerId,
+        callerContext, resource, remoteIp, null);
+  }
 
+  private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
+        ApplicationAttemptId attemptId, ContainerId containerId,
+        CallerContext callerContext, Resource resource, InetAddress remoteIp,
+        RMAuditLogger.ArgsBuilder args) {
     String sLog;
-    if (checkIP) {
+    InetAddress tmpIp = checkIP ? remoteIp : null;
-      sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
+    if (args != null) {
-          attemptId, containerId, resource, callerContext, remoteIp);
+      sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET,
+          tmpIp, args);
     } else {
       sLog = RMAuditLogger.createSuccessLog(USER, OPERATION, TARGET, appId,
-          attemptId, containerId, resource, callerContext, null);
+          attemptId, containerId, resource, callerContext, tmpIp);
     }
     StringBuilder expLog = new StringBuilder();
     expLog.append("USER=test\t");
@@ -165,6 +174,10 @@ private void testSuccessLogFormatHelper(boolean checkIP, ApplicationId appId,
         expLog.append("\tCALLERSIGNATURE=signature");
       }
     }
+    if (args != null) {
+      expLog.append("\tQUEUENAME=root");
+      expLog.append("\tRECURSIVE=true");
+    }
     assertEquals(expLog.toString(), sLog);
   }
 
@@ -241,6 +254,10 @@ private void testSuccessLogFormat(boolean checkIP) {
     testSuccessLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
         new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
             .build(), RESOURCE);
+    RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder()
+        .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
+    testSuccessLogFormatHelper(checkIP, null, null, null, null, null,
+        Server.getRemoteIp(), args);
     testSuccessLogFormatHelperWithIP(checkIP, APPID, ATTEMPTID, CONTAINERID);
     testSuccessLogNulls(checkIP);
   }
@@ -256,9 +273,19 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
   private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
       ApplicationAttemptId attemptId, ContainerId containerId,
       CallerContext callerContext, Resource resource) {
-    String fLog =
+    testFailureLogFormatHelper(checkIP, appId, attemptId, containerId,
+        callerContext, resource, null);
+  }
+
+  private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
+        ApplicationAttemptId attemptId, ContainerId containerId,
+        CallerContext callerContext, Resource resource,
+        RMAuditLogger.ArgsBuilder args) {
+    String fLog = args == null ?
       RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
-      appId, attemptId, containerId, resource, callerContext);
+          appId, attemptId, containerId, resource, callerContext) :
+        RMAuditLogger.createFailureLog(USER, OPERATION, PERM, TARGET, DESC,
+            args);
     StringBuilder expLog = new StringBuilder();
     expLog.append("USER=test\t");
     if (checkIP) {
@@ -288,6 +315,10 @@ private void testFailureLogFormatHelper(boolean checkIP, ApplicationId appId,
         expLog.append("\tCALLERSIGNATURE=signature");
       }
     }
+    if (args != null) {
+      expLog.append("\tQUEUENAME=root");
+      expLog.append("\tRECURSIVE=true");
+    }
     assertEquals(expLog.toString(), fLog);
   }
 
@@ -314,6 +345,10 @@ private void testFailureLogFormat(boolean checkIP) {
     testFailureLogFormatHelper(checkIP, APPID, ATTEMPTID, CONTAINERID,
         new CallerContext.Builder(CALLER_CONTEXT).setSignature(CALLER_SIGNATURE)
             .build(), RESOURCE);
+    RMAuditLogger.ArgsBuilder args = new RMAuditLogger.ArgsBuilder()
+        .append(Keys.QUEUENAME, QUEUE).append(Keys.RECURSIVE, "true");
+    testFailureLogFormatHelper(checkIP, null, null, null, null, null,
+        args);
   }
 
   /**