Use '--cursor-after' flag to get recent journal messages (#51366) (#51445)

When we get Elasticsearch logs from journald, we want to fetch only log
messages from the last run. There are two reasons for this. First, if
there are many logs, we might get a string that's too large for our
utility methods. Second, when we're looking for a specific message or
error, we almost certainly want to look only at messages from the last
execution.

Previously, we've been trying to do this by clearing out the physical
files under the journald process. But there seems to be some contention
over these directories: if journald writes a log file in between when
our deletion command deletes the file and when it deletes the log
directory, the deletion will fail.

Instead, we can use the cursor capablity of journald to retrieve journal
entries that occur only after a certain cursor. This avoids any effort
to interfere with the underlying file operations of journald.

qa/os/src/test/java/org/elasticsearch/packaging/test/PackageTests.java

@@ -22,6 +22,7 @@ package org.elasticsearch.packaging.test;
 import com.carrotsearch.randomizedtesting.generators.RandomStrings;
 import org.apache.http.client.fluent.Request;
 import org.elasticsearch.packaging.util.FileUtils;
+import org.elasticsearch.packaging.util.Packages;
 import org.elasticsearch.packaging.util.Shell.Result;
 import org.junit.BeforeClass;
 
@@ -46,7 +47,6 @@ import static org.elasticsearch.packaging.util.FileUtils.slurp;
 import static org.elasticsearch.packaging.util.Packages.SYSTEMD_SERVICE;
 import static org.elasticsearch.packaging.util.Packages.assertInstalled;
 import static org.elasticsearch.packaging.util.Packages.assertRemoved;
-import static org.elasticsearch.packaging.util.Packages.clearJournal;
 import static org.elasticsearch.packaging.util.Packages.installPackage;
 import static org.elasticsearch.packaging.util.Packages.remove;
 import static org.elasticsearch.packaging.util.Packages.restartElasticsearch;
@@ -343,10 +343,9 @@ public class PackageTests extends PackagingTestCase {
             append(tempConf.resolve("elasticsearch.yml"), "discovery.zen.ping.unicast.hosts:15172.30.5.3416172.30.5.35, 172.30.5.17]\n");
 
             // Make sure we don't pick up the journal entries for previous ES instances.
-            clearJournal(sh);
+            Packages.JournaldWrapper journald = new Packages.JournaldWrapper(sh);
             runElasticsearchStartCommand();
-
-            final Result logs = sh.run("journalctl -u elasticsearch.service");
+            final Result logs = journald.getLogs();
 
             assertThat(logs.stdout, containsString("Failed to load settings from [elasticsearch.yml]"));
         });

qa/os/src/test/java/org/elasticsearch/packaging/util/Packages.java

@@ -280,17 +280,6 @@ public class Packages {
         return sh.runIgnoreExitCode("service elasticsearch start");
     }
 
-    /**
-     * Clears the systemd journal. This is intended to clear the <code>journalctl</code> output
-     * before a test that checks the journal output.
-     */
-    public static void clearJournal(Shell sh) {
-        if (isSystemd()) {
-            sh.run("rm -rf /run/log/journal/");
-            sh.run("systemctl restart systemd-journald");
-        }
-    }
-
     public static void assertElasticsearchStarted(Shell sh, Installation installation) throws Exception {
         waitForElasticsearch(installation);
 
@@ -318,4 +307,41 @@ public class Packages {
         }
         assertElasticsearchStarted(sh, installation);
     }
+
+    /**
+     * A small wrapper for retrieving only recent journald logs for the
+     * Elasticsearch service. It works by creating a cursor for the logs
+     * when instantiated, and advancing that cursor when the {@code clear()}
+     * method is called.
+     */
+    public static class JournaldWrapper {
+        private Shell sh;
+        private String cursor;
+
+        /**
+         * Create a new wrapper for Elasticsearch JournalD logs.
+         * @param sh A shell with appropriate permissions.
+         */
+        public JournaldWrapper(Shell sh) {
+            this.sh = sh;
+            clear();
+        }
+
+        /**
+         * "Clears" the journaled messages by retrieving the latest cursor
+         * for Elasticsearch logs and storing it in class state.
+         */
+        public void clear() {
+            cursor = sh.run("sudo journalctl --unit=elasticsearch.service --lines=0 --show-cursor -o cat" +
+                " | sed -e 's/-- cursor: //'").stdout.trim();
+        }
+
+        /**
+         * Retrieves all log messages coming after the stored cursor.
+         * @return Recent journald logs for the Elasticsearch service.
+         */
+        public Result getLogs() {
+            return sh.run("journalctl -u elasticsearch.service --after-cursor='" + this.cursor + "'");
+        }
+    }
 }