Testclusters: Convert additional projects (#43625)

* Testclusters: Convert additional projects

Found some more that were not using testclusters from elasticsearch-ci/1

* Allow IOException too

* Make the client more resilient

qa/ccs-unavailable-clusters/build.gradle

@@ -16,6 +16,7 @@
  * specific language governing permissions and limitations
  * under the License.
  */
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test-with-dependencies'

qa/die-with-dignity/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.esplugin'
 
 esplugin {
@@ -24,14 +25,17 @@ esplugin {
     classname 'org.elasticsearch.DieWithDignityPlugin'
 }
 
-integTestRunner {
+integTest.runner {
     systemProperty 'tests.security.manager', 'false'
     systemProperty 'tests.system_call_filter', 'false'
-    nonInputProperties.systemProperty 'pidfile', "${-> integTest.getNodes().get(0).pidFile}"
-    nonInputProperties.systemProperty 'log', "${-> integTest.getNodes().get(0).homeDir}/logs/${-> integTest.getNodes().get(0).clusterName}_server.json"
+    nonInputProperties.systemProperty 'log', "${-> testClusters.integTest.singleNode().getServerLog()}"
     systemProperty 'runtime.java.home', "${project.runtimeJavaHome}"
 }
 
+testClusters.integTest {
+    systemProperty "die.with.dignity.test", "whatever"
+}
+
 test.enabled = false
 
 check.dependsOn integTest

qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java

@@ -36,6 +36,10 @@ import java.util.function.Supplier;
 
 public class DieWithDignityPlugin extends Plugin implements ActionPlugin {
 
+    public DieWithDignityPlugin() {
+        assert System.getProperty("die.with.dignity.test") != null : "test should pass the `die.with.dignity.test` property";
+    }
+
     @Override
     public List<RestHandler> getRestHandlers(
             final Settings settings,

qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java

@@ -19,12 +19,10 @@
 
 package org.elasticsearch.qa.die_with_dignity;
 
-import org.apache.http.ConnectionClosedException;
-import org.apache.lucene.util.Constants;
 import org.elasticsearch.client.Request;
 import org.elasticsearch.common.io.PathUtils;
+import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.test.rest.ESRestTestCase;
-import org.hamcrest.Matcher;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -36,51 +34,28 @@ import java.util.Iterator;
 import java.util.List;
 
 import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.hasSize;
-import static org.hamcrest.Matchers.hasToString;
-import static org.hamcrest.Matchers.instanceOf;
 import static org.hamcrest.Matchers.not;
 
 public class DieWithDignityIT extends ESRestTestCase {
     @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43413")
     public void testDieWithDignity() throws Exception {
-        // deleting the PID file prevents stopping the cluster from failing since it occurs if and only if the PID file exists
-        final Path pidFile = PathUtils.get(System.getProperty("pidfile"));
-        final List<String> pidFileLines = Files.readAllLines(pidFile);
-        assertThat(pidFileLines, hasSize(1));
-        final int pid = Integer.parseInt(pidFileLines.get(0));
-        Files.delete(pidFile);
-        IOException e = expectThrows(IOException.class,
-            () -> client().performRequest(new Request("GET", "/_die_with_dignity")));
-        Matcher<IOException> failureMatcher = instanceOf(ConnectionClosedException.class);
-        if (Constants.WINDOWS) {
-            /*
-             * If the other side closes the connection while we're waiting to fill our buffer
-             * we can get IOException with the message below. It seems to only come up on
-             * Windows and it *feels* like it could be a ConnectionClosedException but
-             * upstream does not consider this a bug:
-             * https://issues.apache.org/jira/browse/HTTPASYNC-134
-             *
-             * So we catch it here and consider it "ok".
-             */
-            failureMatcher = either(failureMatcher)
-                .or(hasToString(containsString("An existing connection was forcibly closed by the remote host")));
-        }
-        assertThat(e, failureMatcher);
+        expectThrows(
+            IOException.class,
+            () -> client().performRequest(new Request("GET", "/_die_with_dignity"))
+        );
 
         // the Elasticsearch process should die and disappear from the output of jps
         assertBusy(() -> {
             final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString();
-            final Process process = new ProcessBuilder().command(jpsPath).start();
+            final Process process = new ProcessBuilder().command(jpsPath, "-v").start();
             assertThat(process.waitFor(), equalTo(0));
+
             try (InputStream is = process.getInputStream();
                  BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"))) {
                 String line;
                 while ((line = in.readLine()) != null) {
-                    final int currentPid = Integer.parseInt(line.split("\\s+")[0]);
-                    assertThat(line, pid, not(equalTo(currentPid)));
+                    assertThat(line, line, not(containsString("-Ddie.with.dignity.test")));
                 }
             }
         });
@@ -95,9 +70,9 @@ public class DieWithDignityIT extends ESRestTestCase {
         try {
             while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) {
                 final String line = it.next();
-                if (line.matches(".*ERROR.*o\\.e\\.ExceptionsHelper.*node-0.*fatal error.*")) {
+                if (line.matches(".*ERROR.*o\\.e\\.ExceptionsHelper.*integTest-0.*fatal error.*")) {
                     fatalError = true;
-                } else if (line.matches(".*ERROR.*o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler.*node-0.*"
+                } else if (line.matches(".*ERROR.*o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler.*integTest-0.*"
                     + "fatal error in thread \\[Thread-\\d+\\], exiting.*")) {
                     fatalErrorInThreadExiting = true;
                     assertTrue(it.hasNext());
@@ -127,4 +102,14 @@ public class DieWithDignityIT extends ESRestTestCase {
         return true;
     }
 
+    @Override
+    protected final Settings restClientSettings() {
+        return Settings.builder().put(super.restClientSettings())
+            // increase the timeout here to 90 seconds to handle long waits for a green
+            // cluster health. the waits for green need to be longer than a minute to
+            // account for delayed shards
+            .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "1s")
+            .build();
+    }
+
 }

qa/evil-tests/build.gradle

@@ -23,6 +23,7 @@
  * threads, etc.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {

qa/multi-cluster-search/build.gradle

@@ -19,42 +19,43 @@
 
 import org.elasticsearch.gradle.test.RestIntegTestTask
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-test'
 
 dependencies {
   testCompile project(":client:rest-high-level")
 }
 
-task remoteClusterTest(type: RestIntegTestTask) {
+task 'remote-cluster'(type: RestIntegTestTask) {
   mustRunAfter(precommit)
-}
-
-remoteClusterTestCluster {
-  numNodes = 2
-  clusterName = 'remote-cluster'
-  setting 'cluster.remote.connect', false
-}
-
-remoteClusterTestRunner {
+  runner {
     systemProperty 'tests.rest.suite', 'remote_cluster'
   }
-
-task mixedClusterTest(type: RestIntegTestTask) {}
-
-mixedClusterTestCluster {
-  dependsOn remoteClusterTestRunner
-  setting 'cluster.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\""
-  setting 'cluster.remote.connections_per_cluster', 1
-  setting 'cluster.remote.connect', true
 }
 
-mixedClusterTestRunner {
+testClusters.'remote-cluster' {
+  numberOfNodes = 2
+  setting 'cluster.remote.connect', 'false'
+}
+
+task mixedClusterTest(type: RestIntegTestTask) {
+  useCluster testClusters.'remote-cluster'
+  runner {
+    dependsOn 'remote-cluster'
     systemProperty 'tests.rest.suite', 'multi_cluster'
-  finalizedBy 'remoteClusterTestCluster#node0.stop','remoteClusterTestCluster#node1.stop'
   }
+}
+
+testClusters.mixedClusterTest {
+  setting 'cluster.remote.my_remote_cluster.seeds',
+          { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" }
+  setting 'cluster.remote.connections_per_cluster', '1'
+  setting 'cluster.remote.connect', 'true'
+}
+
 
 task integTest {
-  dependsOn = [mixedClusterTest]
+  dependsOn mixedClusterTest
 }
 
 test.enabled = false // no unit tests for multi-cluster-search, only integration tests

qa/smoke-test-http/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.test-with-dependencies'
@@ -26,7 +27,7 @@ dependencies {
     testCompile project(path: ':plugins:transport-nio', configuration: 'runtime') // for http
 }
 
-integTestRunner {
+integTest.runner {
     /*
      * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each
      * other if we allow them to set the number of available processors as it's set-once in Netty.

qa/smoke-test-ingest-with-all-dependencies/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 
+apply plugin: 'elasticsearch.testclusters'
 apply plugin: 'elasticsearch.standalone-rest-test'
 apply plugin: 'elasticsearch.rest-test'