Add test for dying with dignity (#28987)

I have long wanted an actual test that dying with dignity works. It is
tricky because if dying with dignity works, it means the test JVM dies
which is usually an abnormal condition. And anyway, how does one force a
fatal error to be thrown. I was motivated to investigate this again by
the fact that I missed a backport to one branch leading to an issue
where Elasticsearch would not successfully die with dignity. And now we
have a solution: we install a plugin that throws an out of memory error
when it receives a request. We hack the standalone test infrastructure
to prevent this from failing the test. To do this, we bypass the
security manager and remove the PID file for the node; this tricks the
test infrastructure into thinking that it does not need to stop the
node. We also bypass seccomp so that we can fork jps to make sure that
Elasticsearch really died. And to be extra paranoid, we parse the logs
of the dead Elasticsearch process to make sure it died with
dignity. Never forget.

qa/die-with-dignity/build.gradle

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+apply plugin: 'elasticsearch.esplugin'
+
+esplugin {
+    description 'Out of memory plugin'
+    classname 'org.elasticsearch.DieWithDignityPlugin'
+}
+
+integTestRunner {
+    systemProperty 'tests.security.manager', 'false'
+    systemProperty 'tests.system_call_filter', 'false'
+    systemProperty 'pidfile', "${-> integTest.getNodes().get(0).pidFile}"
+    systemProperty 'log', "${-> integTest.getNodes().get(0).homeDir}/logs/${-> integTest.getNodes().get(0).clusterName}.log"
+    systemProperty 'runtime.java.home', "${project.runtimeJavaHome}"
+}
+
+test.enabled = false
+
+check.dependsOn integTest

qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java

@@ -0,0 +1,51 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.node.DiscoveryNodes;
+import org.elasticsearch.common.settings.ClusterSettings;
+import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestHandler;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Supplier;
+
+public class DieWithDignityPlugin extends Plugin implements ActionPlugin {
+
+    @Override
+    public List<RestHandler> getRestHandlers(
+            final Settings settings,
+            final RestController restController,
+            final ClusterSettings clusterSettings,
+            final IndexScopedSettings indexScopedSettings,
+            final SettingsFilter settingsFilter,
+            final IndexNameExpressionResolver indexNameExpressionResolver,
+            final Supplier<DiscoveryNodes> nodesInCluster) {
+        return Collections.singletonList(new RestDieWithDignityAction(settings, restController));
+    }
+
+}

qa/die-with-dignity/src/main/java/org/elasticsearch/RestDieWithDignityAction.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch;
+
+import org.elasticsearch.client.node.NodeClient;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.http.HttpStats;
+import org.elasticsearch.rest.BaseRestHandler;
+import org.elasticsearch.rest.BytesRestResponse;
+import org.elasticsearch.rest.RestController;
+import org.elasticsearch.rest.RestRequest;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+
+public class RestDieWithDignityAction extends BaseRestHandler {
+
+    RestDieWithDignityAction(final Settings settings, final RestController restController) {
+        super(settings);
+        restController.registerHandler(RestRequest.Method.GET, "/_die_with_dignity", this);
+    }
+
+    @Override
+    public String getName() {
+        return "die_with_dignity_action";
+    }
+
+    @Override
+    protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
+        throw new OutOfMemoryError("die with dignity");
+    }
+
+}

qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.qa.die_with_dignity;
+
+import org.apache.http.ConnectionClosedException;
+import org.elasticsearch.client.Response;
+import org.elasticsearch.client.ResponseListener;
+import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.test.rest.ESRestTestCase;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.instanceOf;
+import static org.hamcrest.Matchers.not;
+
+public class DieWithDignityIT extends ESRestTestCase {
+
+    public void testDieWithDignity() throws Exception {
+        // deleting the PID file prevents stopping the cluster from failing since it occurs if and only if the PID file exists
+        final Path pidFile = PathUtils.get(System.getProperty("pidfile"));
+        final List<String> pidFileLines = Files.readAllLines(pidFile);
+        assertThat(pidFileLines, hasSize(1));
+        final int pid = Integer.parseInt(pidFileLines.get(0));
+        Files.delete(pidFile);
+        expectThrows(ConnectionClosedException.class, () -> client().performRequest("GET", "/_die_with_dignity"));
+
+        // the Elasticsearch process should die and disappear from the output of jps
+        assertBusy(() -> {
+            final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString();
+            final Process process = new ProcessBuilder().command(jpsPath).start();
+            assertThat(process.waitFor(), equalTo(0));
+            try (InputStream is = process.getInputStream();
+                 BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"))) {
+                String line;
+                while ((line = in.readLine()) != null) {
+                    final int currentPid = Integer.parseInt(line.split("\\s+")[0]);
+                    assertThat(line, pid, not(equalTo(currentPid)));
+                }
+            }
+        });
+
+        // parse the logs and ensure that Elasticsearch died with the expected cause
+        final List<String> lines = Files.readAllLines(PathUtils.get(System.getProperty("log")));
+
+        final Iterator<String> it = lines.iterator();
+
+        boolean fatalErrorOnTheNetworkLayer = false;
+        boolean fatalErrorInThreadExiting = false;
+
+        while (it.hasNext() && (fatalErrorOnTheNetworkLayer == false || fatalErrorInThreadExiting == false)) {
+            final String line = it.next();
+            if (line.contains("fatal error on the network layer")) {
+                fatalErrorOnTheNetworkLayer = true;
+            } else if (line.matches(".*\\[ERROR\\]\\[o.e.b.ElasticsearchUncaughtExceptionHandler\\] \\[node-0\\]"
+                    + " fatal error in thread \\[Thread-\\d+\\], exiting$")) {
+                fatalErrorInThreadExiting = true;
+                assertTrue(it.hasNext());
+                assertThat(it.next(), equalTo("java.lang.OutOfMemoryError: die with dignity"));
+            }
+        }
+
+        assertTrue(fatalErrorOnTheNetworkLayer);
+        assertTrue(fatalErrorInThreadExiting);
+    }
+
+    @Override
+    protected boolean preserveClusterUponCompletion() {
+        // as the cluster is dead its state can not be wiped successfully so we have to bypass wiping the cluster
+        return true;
+    }
+
+}

test/framework/src/main/java/org/elasticsearch/bootstrap/BootstrapForTesting.java

@@ -78,7 +78,8 @@ public class BootstrapForTesting {
         }
 
         // just like bootstrap, initialize natives, then SM
-        Bootstrap.initializeNatives(javaTmpDir, true, true, true);
+        final boolean systemCallFilter = Booleans.parseBoolean(System.getProperty("tests.system_call_filter", "true"));
+        Bootstrap.initializeNatives(javaTmpDir, true, systemCallFilter, true);
 
         // initialize probes
         Bootstrap.initializeProbes();

test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java

@@ -145,9 +145,11 @@ public abstract class ESRestTestCase extends ESTestCase {
      */
     @After
     public final void cleanUpCluster() throws Exception {
-        wipeCluster();
-        waitForClusterStateUpdatesToFinish();
-        logIfThereAreRunningTasks();
+        if (preserveClusterUponCompletion() == false) {
+            wipeCluster();
+            waitForClusterStateUpdatesToFinish();
+            logIfThereAreRunningTasks();
+        }
     }
 
     @AfterClass
@@ -175,6 +177,17 @@ public abstract class ESRestTestCase extends ESTestCase {
         return adminClient;
     }
 
+    /**
+     * Returns whether to preserve the state of the cluster upon completion of this test. Defaults to false. If true, overrides the value of
+     * {@link #preserveIndicesUponCompletion()}, {@link #preserveTemplatesUponCompletion()}, {@link #preserveReposUponCompletion()}, and
+     * {@link #preserveSnapshotsUponCompletion()}.
+     *
+     * @return true if the state of the cluster should be preserved
+     */
+    protected boolean preserveClusterUponCompletion() {
+        return false;
+    }
+
     /**
      * Returns whether to preserve the indices created during this test on completion of this test.
      * Defaults to {@code false}. Override this method if indices should be preserved after the test,