From eb1639c5fc7a20b566b55605b33aa988196f982f Mon Sep 17 00:00:00 2001 From: Alpar Torok Date: Wed, 22 May 2019 14:25:54 +0300 Subject: [PATCH] TestClusters: Convert docs (#42100) * TestClusters: Convert docs --- .../gradle/doc/DocsTestPlugin.groovy | 10 ++-- .../testclusters/ElasticsearchCluster.java | 12 +++-- .../testclusters/ElasticsearchNode.java | 11 ++++- .../TestClusterConfiguration.java | 3 ++ .../testclusters/TestClustersPluginIT.java | 28 +++++------ docs/build.gradle | 18 ++++--- docs/reference/cluster/health.asciidoc | 2 +- docs/reference/getting-started.asciidoc | 2 +- x-pack/docs/build.gradle | 49 ++----------------- 9 files changed, 58 insertions(+), 77 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy index a0ce24e45c7..805a1b213e8 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy @@ -18,6 +18,7 @@ */ package org.elasticsearch.gradle.doc +import org.elasticsearch.gradle.OS import org.elasticsearch.gradle.Version import org.elasticsearch.gradle.VersionProperties import org.elasticsearch.gradle.test.ClusterFormationTasks @@ -32,10 +33,13 @@ public class DocsTestPlugin extends RestTestPlugin { @Override public void apply(Project project) { + project.pluginManager.apply('elasticsearch.testclusters') project.pluginManager.apply('elasticsearch.standalone-rest-test') super.apply(project) + String distribution = System.getProperty('tests.distribution', 'default') // The distribution can be configured with -Dtests.distribution on the command line - project.integTestCluster.distribution = System.getProperty('tests.distribution', 'default') + project.testClusters.integTest.distribution = distribution.toUpperCase() + project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") } // Docs are published separately so no need to assemble project.tasks.assemble.enabled = false Map defaultSubstitutions = [ @@ -46,8 +50,8 @@ public class DocsTestPlugin extends RestTestPlugin { '\\{version\\}': Version.fromString(VersionProperties.elasticsearch).toString(), '\\{version_qualified\\}': VersionProperties.elasticsearch, '\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''), - '\\{build_flavor\\}' : project.integTestCluster.distribution, - '\\{build_type\\}' : ClusterFormationTasks.getOs().equals("windows") ? "zip" : "tar", + '\\{build_flavor\\}' : distribution, + '\\{build_type\\}' : OS.conditionalString().onWindows({"zip"}).onUnix({"tar"}).supply(), ] Task listSnippets = project.tasks.create('listSnippets', SnippetsTask) listSnippets.group 'Docs' diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java index 0cb7ee0c10f..e245fb0ead9 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java @@ -42,6 +42,7 @@ import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.BiConsumer; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -70,7 +71,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration { this.nodes = project.container(ElasticsearchNode.class); this.nodes.add( new ElasticsearchNode( - path, clusterName + "-1", + path, clusterName + "-0", services, artifactsExtractDir, workingDirBase ) ); @@ -91,7 +92,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration { ); } - for (int i = nodes.size() + 1 ; i <= numberOfNodes; i++) { + for (int i = nodes.size() ; i < numberOfNodes; i++) { this.nodes.add(new ElasticsearchNode( path, clusterName + "-" + i, services, artifactsExtractDir, workingDirBase )); @@ -99,7 +100,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration { } private ElasticsearchNode getFirstNode() { - return nodes.getAt(clusterName + "-1"); + return nodes.getAt(clusterName + "-0"); } public int getNumberOfNodes() { @@ -276,6 +277,11 @@ public class ElasticsearchCluster implements TestClusterConfiguration { nodes.forEach(each -> each.stop(tailLogs)); } + @Override + public void setNameCustomization(Function nameCustomization) { + nodes.all(each -> each.setNameCustomization(nameCustomization)); + } + @Override public boolean isProcessAlive() { return nodes.stream().noneMatch(node -> node.isProcessAlive() == false); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java index 3bb1fb2ddb6..bba94f6c7d1 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java @@ -50,6 +50,7 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -103,6 +104,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { private String version; private File javaHome; private volatile Process esProcess; + private Function nameCustomization = Function.identity(); ElasticsearchNode(String path, String name, GradleServicesAdapter services, File artifactsExtractDir, File workingDirBase) { this.path = path; @@ -123,7 +125,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { } public String getName() { - return name; + return nameCustomization.apply(name); } public String getVersion() { @@ -536,6 +538,11 @@ public class ElasticsearchNode implements TestClusterConfiguration { esProcess = null; } + @Override + public void setNameCustomization(Function nameCustomizer) { + this.nameCustomization = nameCustomizer; + } + private void stopHandle(ProcessHandle processHandle, boolean forcibly) { // Stop all children first, ES could actually be a child when there's some wrapper process like on Windows. if (processHandle.isAlive() == false) { @@ -656,7 +663,7 @@ public class ElasticsearchNode implements TestClusterConfiguration { } private void createConfiguration() { - defaultConfig.put("node.name", safeName(name)); + defaultConfig.put("node.name", nameCustomization.apply(safeName(name))); defaultConfig.put("path.repo", confPathRepo.toAbsolutePath().toString()); defaultConfig.put("path.data", confPathData.toAbsolutePath().toString()); defaultConfig.put("path.logs", confPathLogs.toAbsolutePath().toString()); diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java index 628dadcbb9d..1ccbeabd4b8 100644 --- a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java +++ b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java @@ -29,6 +29,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.function.Function; import java.util.function.Predicate; import java.util.function.Supplier; @@ -85,6 +86,8 @@ public interface TestClusterConfiguration { void stop(boolean tailLogs); + void setNameCustomization(Function nameSupplier); + default void waitForConditions( LinkedHashMap> waitConditions, long startedAtMillis, diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java index c9086d1459a..39651ff8960 100644 --- a/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java +++ b/buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java @@ -86,8 +86,8 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase { assertOutputContains( result.getOutput(), "> Task :user1", - "Starting `node{::myTestCluster-1}`", - "Stopping `node{::myTestCluster-1}`" + "Starting `node{::myTestCluster-0}`", + "Stopping `node{::myTestCluster-0}`" ); } @@ -104,22 +104,22 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase { assertStartedAndStoppedOnce(result); assertOutputOnlyOnce( result.getOutput(), - "Starting `node{:alpha:myTestCluster-1}`", - "Stopping `node{::myTestCluster-1}`" + "Starting `node{:alpha:myTestCluster-0}`", + "Stopping `node{::myTestCluster-0}`" ); assertOutputOnlyOnce( result.getOutput(), - "Starting `node{::myTestCluster-1}`", - "Stopping `node{:bravo:myTestCluster-1}`" + "Starting `node{::myTestCluster-0}`", + "Stopping `node{:bravo:myTestCluster-0}`" ); } public void testReleased() { BuildResult result = getTestClustersRunner("testReleased").build(); assertTaskSuccessful(result, ":testReleased"); - assertStartedAndStoppedOnce(result, "releasedVersionDefault-1"); - assertStartedAndStoppedOnce(result, "releasedVersionOSS-1"); - assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-1"); + assertStartedAndStoppedOnce(result, "releasedVersionDefault-0"); + assertStartedAndStoppedOnce(result, "releasedVersionOSS-0"); + assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-0"); } public void testIncremental() { @@ -143,7 +143,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase { assertStartedAndStoppedOnce(result); assertOutputContains( result.getOutput(), - "Stopping `node{::myTestCluster-1}`, tailLogs: true", + "Stopping `node{::myTestCluster-0}`, tailLogs: true", "Execution failed for task ':itAlwaysFails'." ); } @@ -155,7 +155,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase { assertStartedAndStoppedOnce(result); assertOutputContains( result.getOutput(), - "Stopping `node{::myTestCluster-1}`, tailLogs: true", + "Stopping `node{::myTestCluster-0}`, tailLogs: true", "Execution failed for task ':itAlwaysFails'." ); } @@ -165,7 +165,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase { assertTaskFailed(result, ":illegalConfigAlter"); assertOutputContains( result.getOutput(), - "Configuration for node{::myTestCluster-1} can not be altered, already locked" + "Configuration for node{::myTestCluster-0} can not be altered, already locked" ); } @@ -173,9 +173,9 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase { public void testMultiNode() { BuildResult result = getTestClustersRunner(":multiNode").build(); assertTaskSuccessful(result, ":multiNode"); + assertStartedAndStoppedOnce(result, "multiNode-0"); assertStartedAndStoppedOnce(result, "multiNode-1"); assertStartedAndStoppedOnce(result, "multiNode-2"); - assertStartedAndStoppedOnce(result, "multiNode-3"); } public void testPluginInstalled() { @@ -211,7 +211,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase { } private void assertStartedAndStoppedOnce(BuildResult result) { - assertStartedAndStoppedOnce(result, "myTestCluster-1"); + assertStartedAndStoppedOnce(result, "myTestCluster-0"); } diff --git a/docs/build.gradle b/docs/build.gradle index 8156d1d54b5..feda444301e 100644 --- a/docs/build.gradle +++ b/docs/build.gradle @@ -1,3 +1,5 @@ +import static org.elasticsearch.gradle.Distribution.DEFAULT + /* * Licensed to Elasticsearch under one or more contributor * license agreements. See the NOTICE file distributed with @@ -35,15 +37,15 @@ buildRestTests.expectedUnconvertedCandidates = [ 'reference/ml/apis/update-snapshot.asciidoc', ] -integTestCluster { - if ("default".equals(integTestCluster.distribution)) { +testClusters.integTest { + if (singleNode().distribution == DEFAULT) { setting 'xpack.license.self_generated.type', 'trial' } // enable regexes in painless so our tests don't complain about example snippets that use them setting 'script.painless.regex.enabled', 'true' Closure configFile = { - extraConfigFile it, "src/test/cluster/config/$it" + extraConfigFile it, file("src/test/cluster/config/$it") } configFile 'analysis/example_word_list.txt' configFile 'analysis/hyphenation_patterns.xml' @@ -52,8 +54,8 @@ integTestCluster { configFile 'userdict_ja.txt' configFile 'userdict_ko.txt' configFile 'KeywordTokenizer.rbbi' - extraConfigFile 'hunspell/en_US/en_US.aff', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff' - extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic' + extraConfigFile 'hunspell/en_US/en_US.aff', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff') + extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic') // Whitelist reindexing from the local node so we can test it. setting 'reindex.remote.whitelist', '127.0.0.1:*' } @@ -65,10 +67,12 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each { if (subproj.path.startsWith(':plugins:repository-')) { return } + // FIXME subproj.afterEvaluate { // need to wait until the project has been configured - integTestCluster { - plugin subproj.path + testClusters.integTest { + plugin file(subproj.bundlePlugin.archiveFile) } + tasks.integTest.dependsOn subproj.bundlePlugin } } diff --git a/docs/reference/cluster/health.asciidoc b/docs/reference/cluster/health.asciidoc index 1e33455d026..d75ce77d1af 100644 --- a/docs/reference/cluster/health.asciidoc +++ b/docs/reference/cluster/health.asciidoc @@ -34,7 +34,7 @@ Returns this: "active_shards_percent_as_number": 50.0 } -------------------------------------------------- -// TESTRESPONSE[s/testcluster/docs_integTestCluster/] +// TESTRESPONSE[s/testcluster/integTest/] // TESTRESPONSE[s/"number_of_pending_tasks" : 0,/"number_of_pending_tasks" : $body.number_of_pending_tasks,/] // TESTRESPONSE[s/"task_max_waiting_in_queue_millis": 0/"task_max_waiting_in_queue_millis": $body.task_max_waiting_in_queue_millis/] diff --git a/docs/reference/getting-started.asciidoc b/docs/reference/getting-started.asciidoc index b81d2b28437..7df9bdfe7aa 100755 --- a/docs/reference/getting-started.asciidoc +++ b/docs/reference/getting-started.asciidoc @@ -301,7 +301,7 @@ And the response: epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent 1475247709 17:01:49 elasticsearch green 1 1 0 0 0 0 0 0 - 100.0% -------------------------------------------------- -// TESTRESPONSE[s/1475247709 17:01:49 elasticsearch/\\d+ \\d+:\\d+:\\d+ docs_integTestCluster/] +// TESTRESPONSE[s/1475247709 17:01:49 elasticsearch/\\d+ \\d+:\\d+:\\d+ integTest/] // TESTRESPONSE[s/0 0 -/0 \\d+ -/] // TESTRESPONSE[_cat] diff --git a/x-pack/docs/build.gradle b/x-pack/docs/build.gradle index 5e56414afed..d33ccddcf50 100644 --- a/x-pack/docs/build.gradle +++ b/x-pack/docs/build.gradle @@ -24,53 +24,13 @@ dependencies { testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts') } -Closure waitWithAuth = { NodeInfo node, AntBuilder ant -> - File tmpFile = new File(node.cwd, 'wait.success') - // wait up to twenty seconds - final long stopTime = System.currentTimeMillis() + 20000L; - Exception lastException = null; - while (System.currentTimeMillis() < stopTime) { - lastException = null; - // we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned - HttpURLConnection httpURLConnection = null; - try { - httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health").openConnection(); - httpURLConnection.setRequestProperty("Authorization", "Basic " + - Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8))); - httpURLConnection.setRequestMethod("GET"); - httpURLConnection.setConnectTimeout(1000); - httpURLConnection.setReadTimeout(30000); - httpURLConnection.connect(); - if (httpURLConnection.getResponseCode() == 200) { - tmpFile.withWriter StandardCharsets.UTF_8.name(), { - it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name())) - } - break; - } - } catch (Exception e) { - logger.debug("failed to call cluster health", e) - lastException = e - } finally { - if (httpURLConnection != null) { - httpURLConnection.disconnect(); - } - } - - // did not start, so wait a bit before trying again - Thread.sleep(500L); - } - if (tmpFile.exists() == false && lastException != null) { - logger.error("final attempt of calling cluster health failed", lastException) - } - return tmpFile.exists() -} - // copy xpack rest api File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources') project.copyRestSpec.from(xpackResources) { include 'rest-api-spec/api/**' } -integTestCluster { + +testClusters.integTest { setting 'xpack.security.enabled', 'true' setting 'xpack.security.authc.api_key.enabled', 'true' setting 'xpack.security.authc.token.enabled', 'true' @@ -78,13 +38,10 @@ integTestCluster { setting 'xpack.monitoring.exporters._local.type', 'local' setting 'xpack.monitoring.exporters._local.enabled', 'false' setting 'xpack.license.self_generated.type', 'trial' - setupCommand 'setupTestAdmin', - 'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser' - waitCondition = waitWithAuth + user username: 'test_admin' } - buildRestTests.docs = fileTree(projectDir) { // No snippets in here! exclude 'build.gradle'