Account for a possible rolled over file while reading the audit log file (#34909)

(cherry picked from commit 75cb6b38ed67dc9d32c9291b0c174ffa94e473bc)

x-pack/plugin/sql/qa/security/build.gradle

@@ -41,8 +41,11 @@ subprojects {
   }
 
   integTestRunner {
+    def today = new Date().format('yyyy-MM-dd')
     systemProperty 'tests.audit.logfile',
         "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_audit.json"
+    systemProperty 'tests.audit.yesterday.logfile',
+        "${ -> integTest.nodes[0].homeDir}/logs/${ -> integTest.nodes[0].clusterName }_audit-${today}.json"
   }
 
   runqa {

x-pack/plugin/sql/qa/security/src/test/java/org/elasticsearch/xpack/sql/qa/security/SqlSecurityTestCase.java

@@ -84,6 +84,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
      * {@code plugin-security.policy}. So we may as well have gradle set the property.
      */
     private static final Path AUDIT_LOG_FILE = lookupAuditLog();
+    private static final Path ROLLED_OVER_AUDIT_LOG_FILE = lookupRolledOverAuditLog();
 
     @SuppressForbidden(reason="security doesn't work with mock filesystem")
     private static Path lookupAuditLog() {
@@ -96,6 +97,16 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
         return Paths.get(auditLogFileString);
     }
     
+    @SuppressForbidden(reason="security doesn't work with mock filesystem")
+    private static Path lookupRolledOverAuditLog() {
+        String auditLogFileString = System.getProperty("tests.audit.yesterday.logfile");
+        if (null == auditLogFileString) {
+            throw new IllegalStateException("tests.audit.yesterday.logfile must be set to run this test. It should be automatically "
+                    + "set by gradle.");
+        }        
+        return Paths.get(auditLogFileString);
+    }
+
     private static boolean oneTimeSetup = false;
     private static boolean auditFailure = false;
 
@@ -107,7 +118,12 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
     /**
      * How much of the audit log was written before the test started.
      */
-    private long auditLogWrittenBeforeTestStart;
+    private static long auditLogWrittenBeforeTestStart;
+    
+    /**
+     * If the audit log file rolled over. This is a rare case possible only at midnight.
+     */
+    private static boolean auditFileRolledOver = false;
 
     public SqlSecurityTestCase(Actions actions) {
         this.actions = actions;
@@ -556,18 +572,51 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
                     if (sm != null) {
                         sm.checkPermission(new SpecialPermission());
                     }
-                    BufferedReader logReader = AccessController.doPrivileged((PrivilegedAction<BufferedReader>) () -> {
+                    
+                    BufferedReader[] logReaders = new BufferedReader[2];
+                    AccessController.doPrivileged((PrivilegedAction<Void>) () -> {
                         try {
-                            return  Files.newBufferedReader(AUDIT_LOG_FILE, StandardCharsets.UTF_8);
+                            // the audit log file rolled over during the test
+                            // and we need to consume the rest of the rolled over file plus the new audit log file
+                            if (auditFileRolledOver == false && Files.exists(ROLLED_OVER_AUDIT_LOG_FILE)) {
+                                // once the audit file rolled over, it will stay like this
+                                auditFileRolledOver = true;
+                                // the order in the array matters, as the readers will be used in that order
+                                logReaders[0] = Files.newBufferedReader(ROLLED_OVER_AUDIT_LOG_FILE, StandardCharsets.UTF_8);
+                            }
+                            logReaders[1] = Files.newBufferedReader(AUDIT_LOG_FILE, StandardCharsets.UTF_8);
+                            return null;
                         } catch (IOException e) {
                             throw new RuntimeException(e);
                         }
                     });
-                    logReader.skip(auditLogWrittenBeforeTestStart);
+                    
+                    // The "index" is used as a way of reading from both rolled over file and current audit file in order: rolled over file
+                    // first, then the audit log file. Very rarely we will read from the rolled over file: when the test happened to run
+                    // at midnight and the audit file rolled over during the test.
+                    int index;
+                    if (logReaders[0] != null) {
+                        logReaders[0].skip(auditLogWrittenBeforeTestStart);
+                        // start with the rolled over file first
+                        index = 0;
+                    } else {
+                        // the current audit log file reader should always be non-null
+                        logReaders[1].skip(auditLogWrittenBeforeTestStart);
+                        // start with the current audit logging file
+                        index = 1;
+                    }
 
                     List<Map<String, Object>> logs = new ArrayList<>();
                     String line;
-                    while ((line = logReader.readLine()) != null) {
+                    while (index < 2) {
+                        line = logReaders[index].readLine();
+                        // when the end of the file is reached, either stop or move to the next reader
+                        if (line == null) {
+                            if (++index == 2) {
+                                break;
+                            }
+                        }
+                        else {
                             try {
                                 final Map<String, Object> log = XContentHelper.convertToMap(JsonXContent.jsonXContent, line, false);
                                 if (false == ("access_denied".equals(log.get("event.action"))
@@ -603,6 +652,7 @@ public abstract class SqlSecurityTestCase extends ESRestTestCase {
                                 throw new IllegalArgumentException("Unrecognized log: " + line, e);
                             }
                         }
+                    }
                     List<Map<String, Object>> allLogs = new ArrayList<>(logs);
                     List<Integer> notMatching = new ArrayList<>();
                     checker: for (int c = 0; c < logCheckers.size(); c++) {

x-pack/plugin/sql/qa/security/src/test/resources/plugin-security.policy

@@ -1,6 +1,7 @@
 grant {
   // Needed to read the audit log file
   permission java.io.FilePermission "${tests.audit.logfile}", "read";
+  permission java.io.FilePermission "${tests.audit.yesterday.logfile}", "read";
 
   //// Required by ssl subproject:
   // Required for the net client to setup ssl rather than use global ssl.