From 23f739b513710878b1e95e1693d4b0a1d565aa06 Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Fri, 28 Jun 2019 16:38:17 +0300 Subject: [PATCH] Testclusters: Convert additional projects (#43625) * Testclusters: Convert additional projects Found some more that were not using testclusters from elasticsearch-ci/1 * Allow IOException too * Make the client more resilient --- qa/ccs-unavailable-clusters/build.gradle | 1 + qa/die-with-dignity/build.gradle | 10 +++- .../elasticsearch/DieWithDignityPlugin.java | 4 ++ .../qa/die_with_dignity/DieWithDignityIT.java | 55 +++++++------------ qa/evil-tests/build.gradle | 1 + qa/multi-cluster-search/build.gradle | 39 ++++++------- qa/smoke-test-http/build.gradle | 3 +- .../build.gradle | 1 + 8 files changed, 56 insertions(+), 58 deletions(-) diff --git a/qa/ccs-unavailable-clusters/build.gradle b/qa/ccs-unavailable-clusters/build.gradle index ea80ee983b8..749623b26c3 100644 --- a/qa/ccs-unavailable-clusters/build.gradle +++ b/qa/ccs-unavailable-clusters/build.gradle @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.test-with-dependencies' diff --git a/qa/die-with-dignity/build.gradle b/qa/die-with-dignity/build.gradle index 140df6e283a..a40f6366e65 100644 --- a/qa/die-with-dignity/build.gradle +++ b/qa/die-with-dignity/build.gradle @@ -17,6 +17,7 @@ * under the License. */ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.esplugin' esplugin { @@ -24,14 +25,17 @@ esplugin { classname 'org.elasticsearch.DieWithDignityPlugin' } -integTestRunner { +integTest.runner { systemProperty 'tests.security.manager', 'false' systemProperty 'tests.system_call_filter', 'false' - nonInputProperties.systemProperty 'pidfile', "${-> integTest.getNodes().get(0).pidFile}" - nonInputProperties.systemProperty 'log', "${-> integTest.getNodes().get(0).homeDir}/logs/${-> integTest.getNodes().get(0).clusterName}_server.json" + nonInputProperties.systemProperty 'log', "${-> testClusters.integTest.singleNode().getServerLog()}" systemProperty 'runtime.java.home', "${project.runtimeJavaHome}" } +testClusters.integTest { + systemProperty "die.with.dignity.test", "whatever" +} + test.enabled = false check.dependsOn integTest diff --git a/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java b/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java index ed1e3d3879a..8027eeb8948 100644 --- a/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java +++ b/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java @@ -36,6 +36,10 @@ import java.util.function.Supplier; public class DieWithDignityPlugin extends Plugin implements ActionPlugin { + public DieWithDignityPlugin() { + assert System.getProperty("die.with.dignity.test") != null : "test should pass the `die.with.dignity.test` property"; + } + @Override public List getRestHandlers( final Settings settings, diff --git a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java index 737106ee920..20ceb90ca03 100644 --- a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java +++ b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java @@ -19,12 +19,10 @@ package org.elasticsearch.qa.die_with_dignity; -import org.apache.http.ConnectionClosedException; -import org.apache.lucene.util.Constants; import org.elasticsearch.client.Request; import org.elasticsearch.common.io.PathUtils; +import org.elasticsearch.common.settings.Settings; import org.elasticsearch.test.rest.ESRestTestCase; -import org.hamcrest.Matcher; import java.io.BufferedReader; import java.io.IOException; @@ -36,51 +34,28 @@ import java.util.Iterator; import java.util.List; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.either; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasSize; -import static org.hamcrest.Matchers.hasToString; -import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; public class DieWithDignityIT extends ESRestTestCase { @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/43413") public void testDieWithDignity() throws Exception { - // deleting the PID file prevents stopping the cluster from failing since it occurs if and only if the PID file exists - final Path pidFile = PathUtils.get(System.getProperty("pidfile")); - final List pidFileLines = Files.readAllLines(pidFile); - assertThat(pidFileLines, hasSize(1)); - final int pid = Integer.parseInt(pidFileLines.get(0)); - Files.delete(pidFile); - IOException e = expectThrows(IOException.class, - () -> client().performRequest(new Request("GET", "/_die_with_dignity"))); - Matcher failureMatcher = instanceOf(ConnectionClosedException.class); - if (Constants.WINDOWS) { - /* - * If the other side closes the connection while we're waiting to fill our buffer - * we can get IOException with the message below. It seems to only come up on - * Windows and it *feels* like it could be a ConnectionClosedException but - * upstream does not consider this a bug: - * https://issues.apache.org/jira/browse/HTTPASYNC-134 - * - * So we catch it here and consider it "ok". - */ - failureMatcher = either(failureMatcher) - .or(hasToString(containsString("An existing connection was forcibly closed by the remote host"))); - } - assertThat(e, failureMatcher); + expectThrows( + IOException.class, + () -> client().performRequest(new Request("GET", "/_die_with_dignity")) + ); // the Elasticsearch process should die and disappear from the output of jps assertBusy(() -> { final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString(); - final Process process = new ProcessBuilder().command(jpsPath).start(); + final Process process = new ProcessBuilder().command(jpsPath, "-v").start(); assertThat(process.waitFor(), equalTo(0)); + try (InputStream is = process.getInputStream(); BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"))) { String line; while ((line = in.readLine()) != null) { - final int currentPid = Integer.parseInt(line.split("\\s+")[0]); - assertThat(line, pid, not(equalTo(currentPid))); + assertThat(line, line, not(containsString("-Ddie.with.dignity.test"))); } } }); @@ -95,9 +70,9 @@ public class DieWithDignityIT extends ESRestTestCase { try { while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) { final String line = it.next(); - if (line.matches(".*ERROR.*o\\.e\\.ExceptionsHelper.*node-0.*fatal error.*")) { + if (line.matches(".*ERROR.*o\\.e\\.ExceptionsHelper.*integTest-0.*fatal error.*")) { fatalError = true; - } else if (line.matches(".*ERROR.*o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler.*node-0.*" + } else if (line.matches(".*ERROR.*o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler.*integTest-0.*" + "fatal error in thread \\[Thread-\\d+\\], exiting.*")) { fatalErrorInThreadExiting = true; assertTrue(it.hasNext()); @@ -127,4 +102,14 @@ public class DieWithDignityIT extends ESRestTestCase { return true; } + @Override + protected final Settings restClientSettings() { + return Settings.builder().put(super.restClientSettings()) + // increase the timeout here to 90 seconds to handle long waits for a green + // cluster health. the waits for green need to be longer than a minute to + // account for delayed shards + .put(ESRestTestCase.CLIENT_SOCKET_TIMEOUT, "1s") + .build(); + } + } diff --git a/qa/evil-tests/build.gradle b/qa/evil-tests/build.gradle index 2f9239e5c2f..38c1b3e1a9a 100644 --- a/qa/evil-tests/build.gradle +++ b/qa/evil-tests/build.gradle @@ -23,6 +23,7 @@ * threads, etc. */ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' dependencies { diff --git a/qa/multi-cluster-search/build.gradle b/qa/multi-cluster-search/build.gradle index 7f923d03f71..1913c86fc9c 100644 --- a/qa/multi-cluster-search/build.gradle +++ b/qa/multi-cluster-search/build.gradle @@ -19,42 +19,43 @@ import org.elasticsearch.gradle.test.RestIntegTestTask +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-test' dependencies { testCompile project(":client:rest-high-level") } -task remoteClusterTest(type: RestIntegTestTask) { +task 'remote-cluster'(type: RestIntegTestTask) { mustRunAfter(precommit) + runner { + systemProperty 'tests.rest.suite', 'remote_cluster' + } } -remoteClusterTestCluster { - numNodes = 2 - clusterName = 'remote-cluster' - setting 'cluster.remote.connect', false +testClusters.'remote-cluster' { + numberOfNodes = 2 + setting 'cluster.remote.connect', 'false' } -remoteClusterTestRunner { - systemProperty 'tests.rest.suite', 'remote_cluster' +task mixedClusterTest(type: RestIntegTestTask) { + useCluster testClusters.'remote-cluster' + runner { + dependsOn 'remote-cluster' + systemProperty 'tests.rest.suite', 'multi_cluster' + } } -task mixedClusterTest(type: RestIntegTestTask) {} - -mixedClusterTestCluster { - dependsOn remoteClusterTestRunner - setting 'cluster.remote.my_remote_cluster.seeds', "\"${-> remoteClusterTest.nodes.get(0).transportUri()}\"" - setting 'cluster.remote.connections_per_cluster', 1 - setting 'cluster.remote.connect', true +testClusters.mixedClusterTest { + setting 'cluster.remote.my_remote_cluster.seeds', + { "\"${testClusters.'remote-cluster'.getAllTransportPortURI().get(0)}\"" } + setting 'cluster.remote.connections_per_cluster', '1' + setting 'cluster.remote.connect', 'true' } -mixedClusterTestRunner { - systemProperty 'tests.rest.suite', 'multi_cluster' - finalizedBy 'remoteClusterTestCluster#node0.stop','remoteClusterTestCluster#node1.stop' -} task integTest { - dependsOn = [mixedClusterTest] + dependsOn mixedClusterTest } test.enabled = false // no unit tests for multi-cluster-search, only integration tests diff --git a/qa/smoke-test-http/build.gradle b/qa/smoke-test-http/build.gradle index ef1a97fc7ab..90fc9e39501 100644 --- a/qa/smoke-test-http/build.gradle +++ b/qa/smoke-test-http/build.gradle @@ -17,6 +17,7 @@ * under the License. */ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test' apply plugin: 'elasticsearch.test-with-dependencies' @@ -26,7 +27,7 @@ dependencies { testCompile project(path: ':plugins:transport-nio', configuration: 'runtime') // for http } -integTestRunner { +integTest.runner { /* * We have to disable setting the number of available processors as tests in the same JVM randomize processors and will step on each * other if we allow them to set the number of available processors as it's set-once in Netty. diff --git a/qa/smoke-test-ingest-with-all-dependencies/build.gradle b/qa/smoke-test-ingest-with-all-dependencies/build.gradle index 9267f90cd7e..9f5c40ac937 100644 --- a/qa/smoke-test-ingest-with-all-dependencies/build.gradle +++ b/qa/smoke-test-ingest-with-all-dependencies/build.gradle @@ -17,6 +17,7 @@ * under the License. */ +apply plugin: 'elasticsearch.testclusters' apply plugin: 'elasticsearch.standalone-rest-test' apply plugin: 'elasticsearch.rest-test'