honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

TestClusters: Convert docs (#42100) 

* TestClusters: Convert docs
This commit is contained in:
Alpar Torok 2019-05-22 14:25:54 +03:00
parent f338005179
commit eb1639c5fc
9 changed files with 58 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
View File

@ -18,6 +18,7 @@
*/
package org.elasticsearch.gradle.doc
import org.elasticsearch.gradle.OS
import org.elasticsearch.gradle.Version
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.test.ClusterFormationTasks
@ -32,10 +33,13 @@ public class DocsTestPlugin extends RestTestPlugin {
@Override
public void apply(Project project) {
project.pluginManager.apply('elasticsearch.testclusters')
project.pluginManager.apply('elasticsearch.standalone-rest-test')
super.apply(project)
String distribution = System.getProperty('tests.distribution', 'default')
// The distribution can be configured with -Dtests.distribution on the command line
project.integTestCluster.distribution = System.getProperty('tests.distribution', 'default')
project.testClusters.integTest.distribution = distribution.toUpperCase()
project.testClusters.integTest.nameCustomization = { it.replace("integTest", "node") }
// Docs are published separately so no need to assemble
project.tasks.assemble.enabled = false
Map<String, String> defaultSubstitutions = [
@ -46,8 +50,8 @@ public class DocsTestPlugin extends RestTestPlugin {
'\\{version\\}': Version.fromString(VersionProperties.elasticsearch).toString(),
'\\{version_qualified\\}': VersionProperties.elasticsearch,
'\\{lucene_version\\}' : VersionProperties.lucene.replaceAll('-snapshot-\\w+$', ''),
'\\{build_flavor\\}' : project.integTestCluster.distribution,
'\\{build_type\\}' : ClusterFormationTasks.getOs().equals("windows") ? "zip" : "tar",
'\\{build_flavor\\}' : distribution,
'\\{build_type\\}' : OS.conditionalString().onWindows({"zip"}).onUnix({"tar"}).supply(),
]
Task listSnippets = project.tasks.create('listSnippets', SnippetsTask)
listSnippets.group 'Docs'

12
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
View File

@ -42,6 +42,7 @@ import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -70,7 +71,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
this.nodes = project.container(ElasticsearchNode.class);
this.nodes.add(
new ElasticsearchNode(
path, clusterName + "-1",
path, clusterName + "-0",
services, artifactsExtractDir, workingDirBase
)
);
@ -91,7 +92,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
);
}
for (int i = nodes.size() + 1 ; i <= numberOfNodes; i++) {
for (int i = nodes.size() ; i < numberOfNodes; i++) {
this.nodes.add(new ElasticsearchNode(
path, clusterName + "-" + i, services, artifactsExtractDir, workingDirBase
));
@ -99,7 +100,7 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
}
private ElasticsearchNode getFirstNode() {
return nodes.getAt(clusterName + "-1");
return nodes.getAt(clusterName + "-0");
}
public int getNumberOfNodes() {
@ -276,6 +277,11 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
nodes.forEach(each -> each.stop(tailLogs));
}
@Override
public void setNameCustomization(Function<String, String> nameCustomization) {
nodes.all(each -> each.setNameCustomization(nameCustomization));
}
@Override
public boolean isProcessAlive() {
return nodes.stream().noneMatch(node -> node.isProcessAlive() == false);

11
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View File

@ -50,6 +50,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@ -103,6 +104,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private String version;
private File javaHome;
private volatile Process esProcess;
private Function<String, String> nameCustomization = Function.identity();
ElasticsearchNode(String path, String name, GradleServicesAdapter services, File artifactsExtractDir, File workingDirBase) {
this.path = path;
@ -123,7 +125,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
public String getName() {
return name;
return nameCustomization.apply(name);
}
public String getVersion() {
@ -536,6 +538,11 @@ public class ElasticsearchNode implements TestClusterConfiguration {
esProcess = null;
}
@Override
public void setNameCustomization(Function<String, String> nameCustomizer) {
this.nameCustomization = nameCustomizer;
}
private void stopHandle(ProcessHandle processHandle, boolean forcibly) {
// Stop all children first, ES could actually be a child when there's some wrapper process like on Windows.
if (processHandle.isAlive() == false) {
@ -656,7 +663,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
private void createConfiguration() {
defaultConfig.put("node.name", safeName(name));
defaultConfig.put("node.name", nameCustomization.apply(safeName(name)));
defaultConfig.put("path.repo", confPathRepo.toAbsolutePath().toString());
defaultConfig.put("path.data", confPathData.toAbsolutePath().toString());
defaultConfig.put("path.logs", confPathLogs.toAbsolutePath().toString());

3
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
View File

@ -29,6 +29,7 @@ import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.Supplier;
@ -85,6 +86,8 @@ public interface TestClusterConfiguration {
void stop(boolean tailLogs);
void setNameCustomization(Function<String, String> nameSupplier);
default void waitForConditions(
LinkedHashMap<String, Predicate<TestClusterConfiguration>> waitConditions,
long startedAtMillis,

28
buildSrc/src/test/java/org/elasticsearch/gradle/testclusters/TestClustersPluginIT.java
View File

@ -86,8 +86,8 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
assertOutputContains(
result.getOutput(),
"> Task :user1",
"Starting `node{::myTestCluster-1}`",
"Stopping `node{::myTestCluster-1}`"
"Starting `node{::myTestCluster-0}`",
"Stopping `node{::myTestCluster-0}`"
);
}
@ -104,22 +104,22 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
assertStartedAndStoppedOnce(result);
assertOutputOnlyOnce(
result.getOutput(),
"Starting `node{:alpha:myTestCluster-1}`",
"Stopping `node{::myTestCluster-1}`"
"Starting `node{:alpha:myTestCluster-0}`",
"Stopping `node{::myTestCluster-0}`"
);
assertOutputOnlyOnce(
result.getOutput(),
"Starting `node{::myTestCluster-1}`",
"Stopping `node{:bravo:myTestCluster-1}`"
"Starting `node{::myTestCluster-0}`",
"Stopping `node{:bravo:myTestCluster-0}`"
);
}
public void testReleased() {
BuildResult result = getTestClustersRunner("testReleased").build();
assertTaskSuccessful(result, ":testReleased");
assertStartedAndStoppedOnce(result, "releasedVersionDefault-1");
assertStartedAndStoppedOnce(result, "releasedVersionOSS-1");
assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-1");
assertStartedAndStoppedOnce(result, "releasedVersionDefault-0");
assertStartedAndStoppedOnce(result, "releasedVersionOSS-0");
assertStartedAndStoppedOnce(result, "releasedVersionIntegTest-0");
}
public void testIncremental() {
@ -143,7 +143,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
assertStartedAndStoppedOnce(result);
assertOutputContains(
result.getOutput(),
"Stopping `node{::myTestCluster-1}`, tailLogs: true",
"Stopping `node{::myTestCluster-0}`, tailLogs: true",
"Execution failed for task ':itAlwaysFails'."
);
}
@ -155,7 +155,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
assertStartedAndStoppedOnce(result);
assertOutputContains(
result.getOutput(),
"Stopping `node{::myTestCluster-1}`, tailLogs: true",
"Stopping `node{::myTestCluster-0}`, tailLogs: true",
"Execution failed for task ':itAlwaysFails'."
);
}
@ -165,7 +165,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
assertTaskFailed(result, ":illegalConfigAlter");
assertOutputContains(
result.getOutput(),
"Configuration for node{::myTestCluster-1} can not be altered, already locked"
"Configuration for node{::myTestCluster-0} can not be altered, already locked"
);
}
@ -173,9 +173,9 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
public void testMultiNode() {
BuildResult result = getTestClustersRunner(":multiNode").build();
assertTaskSuccessful(result, ":multiNode");
assertStartedAndStoppedOnce(result, "multiNode-0");
assertStartedAndStoppedOnce(result, "multiNode-1");
assertStartedAndStoppedOnce(result, "multiNode-2");
assertStartedAndStoppedOnce(result, "multiNode-3");
}
public void testPluginInstalled() {
@ -211,7 +211,7 @@ public class TestClustersPluginIT extends GradleIntegrationTestCase {
}
private void assertStartedAndStoppedOnce(BuildResult result) {
assertStartedAndStoppedOnce(result, "myTestCluster-1");
assertStartedAndStoppedOnce(result, "myTestCluster-0");
}

18
docs/build.gradle
View File

@ -1,3 +1,5 @@
import static org.elasticsearch.gradle.Distribution.DEFAULT
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
@ -35,15 +37,15 @@ buildRestTests.expectedUnconvertedCandidates = [
'reference/ml/apis/update-snapshot.asciidoc',
]
integTestCluster {
if ("default".equals(integTestCluster.distribution)) {
testClusters.integTest {
if (singleNode().distribution == DEFAULT) {
setting 'xpack.license.self_generated.type', 'trial'
}
// enable regexes in painless so our tests don't complain about example snippets that use them
setting 'script.painless.regex.enabled', 'true'
Closure configFile = {
extraConfigFile it, "src/test/cluster/config/$it"
extraConfigFile it, file("src/test/cluster/config/$it")
}
configFile 'analysis/example_word_list.txt'
configFile 'analysis/hyphenation_patterns.xml'
@ -52,8 +54,8 @@ integTestCluster {
configFile 'userdict_ja.txt'
configFile 'userdict_ko.txt'
configFile 'KeywordTokenizer.rbbi'
extraConfigFile 'hunspell/en_US/en_US.aff', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff'
extraConfigFile 'hunspell/en_US/en_US.dic', '../server/src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic'
extraConfigFile 'hunspell/en_US/en_US.aff', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.aff')
extraConfigFile 'hunspell/en_US/en_US.dic', project(":server").file('src/test/resources/indices/analyze/conf_dir/hunspell/en_US/en_US.dic')
// Whitelist reindexing from the local node so we can test it.
setting 'reindex.remote.whitelist', '127.0.0.1:*'
}
@ -65,10 +67,12 @@ project.rootProject.subprojects.findAll { it.parent.path == ':plugins' }.each {
if (subproj.path.startsWith(':plugins:repository-')) {
return
}
// FIXME
subproj.afterEvaluate { // need to wait until the project has been configured
integTestCluster {
plugin subproj.path
testClusters.integTest {
plugin file(subproj.bundlePlugin.archiveFile)
}
tasks.integTest.dependsOn subproj.bundlePlugin
}
}

2
docs/reference/cluster/health.asciidoc
View File

@ -34,7 +34,7 @@ Returns this:
"active_shards_percent_as_number": 50.0
}
--------------------------------------------------
// TESTRESPONSE[s/testcluster/docs_integTestCluster/]
// TESTRESPONSE[s/testcluster/integTest/]
// TESTRESPONSE[s/"number_of_pending_tasks" : 0,/"number_of_pending_tasks" : $body.number_of_pending_tasks,/]
// TESTRESPONSE[s/"task_max_waiting_in_queue_millis": 0/"task_max_waiting_in_queue_millis": $body.task_max_waiting_in_queue_millis/]

2
docs/reference/getting-started.asciidoc
View File

@ -301,7 +301,7 @@ And the response:
epoch timestamp cluster status node.total node.data shards pri relo init unassign pending_tasks max_task_wait_time active_shards_percent
1475247709 17:01:49 elasticsearch green 1 1 0 0 0 0 0 0 - 100.0%
--------------------------------------------------
// TESTRESPONSE[s/1475247709 17:01:49 elasticsearch/\\d+ \\d+:\\d+:\\d+ docs_integTestCluster/]
// TESTRESPONSE[s/1475247709 17:01:49 elasticsearch/\\d+ \\d+:\\d+:\\d+ integTest/]
// TESTRESPONSE[s/0 0 -/0 \\d+ -/]
// TESTRESPONSE[_cat]

49
x-pack/docs/build.gradle
View File

@ -24,53 +24,13 @@ dependencies {
testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
}
Closure waitWithAuth = { NodeInfo node, AntBuilder ant ->
File tmpFile = new File(node.cwd, 'wait.success')
// wait up to twenty seconds
final long stopTime = System.currentTimeMillis() + 20000L;
Exception lastException = null;
while (System.currentTimeMillis() < stopTime) {
lastException = null;
// we use custom wait logic here as the elastic user is not available immediately and ant.get will fail when a 401 is returned
HttpURLConnection httpURLConnection = null;
try {
httpURLConnection = (HttpURLConnection) new URL("http://${node.httpUri()}/_cluster/health").openConnection();
httpURLConnection.setRequestProperty("Authorization", "Basic " +
Base64.getEncoder().encodeToString("test_admin:x-pack-test-password".getBytes(StandardCharsets.UTF_8)));
httpURLConnection.setRequestMethod("GET");
httpURLConnection.setConnectTimeout(1000);
httpURLConnection.setReadTimeout(30000);
httpURLConnection.connect();
if (httpURLConnection.getResponseCode() == 200) {
tmpFile.withWriter StandardCharsets.UTF_8.name(), {
it.write(httpURLConnection.getInputStream().getText(StandardCharsets.UTF_8.name()))
}
break;
}
} catch (Exception e) {
logger.debug("failed to call cluster health", e)
lastException = e
} finally {
if (httpURLConnection != null) {
httpURLConnection.disconnect();
}
}
// did not start, so wait a bit before trying again
Thread.sleep(500L);
}
if (tmpFile.exists() == false && lastException != null) {
logger.error("final attempt of calling cluster health failed", lastException)
}
return tmpFile.exists()
}
// copy xpack rest api
File xpackResources = new File(xpackProject('plugin').projectDir, 'src/test/resources')
project.copyRestSpec.from(xpackResources) {
include 'rest-api-spec/api/**'
}
integTestCluster {
testClusters.integTest {
setting 'xpack.security.enabled', 'true'
setting 'xpack.security.authc.api_key.enabled', 'true'
setting 'xpack.security.authc.token.enabled', 'true'
@ -78,13 +38,10 @@ integTestCluster {
setting 'xpack.monitoring.exporters._local.type', 'local'
setting 'xpack.monitoring.exporters._local.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setupCommand 'setupTestAdmin',
'bin/elasticsearch-users', 'useradd', 'test_admin', '-p', 'x-pack-test-password', '-r', 'superuser'
waitCondition = waitWithAuth
user username: 'test_admin'
}
buildRestTests.docs = fileTree(projectDir) {
// No snippets in here!
exclude 'build.gradle'