honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Testclusters: convert ccr tests (#42313)

This commit is contained in:
Alpar Torok 2019-05-29 16:16:04 +03:00
parent 43665183c2
commit 4ba94a5051
13 changed files with 283 additions and 215 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
View File

@ -79,7 +79,10 @@ class RestIntegTestTask extends DefaultTask {
// disable the build cache for rest test tasks
// there are a number of inputs we aren't properly tracking here so we'll just not cache these for now
runner.outputs.doNotCacheIf('Caching is disabled for REST integration tests') { true }
runner.getOutputs().doNotCacheIf(
"Caching is disabled for REST integration tests",
{ false }
);
// override/add more for rest tests
runner.maxParallelForks = 1
@ -282,4 +285,5 @@ class RestIntegTestTask extends DefaultTask {
}
return copyRestSpec
}
}

15
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java
View File

@ -186,6 +186,16 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
nodes.all(each -> each.environment(key, valueSupplier));
}
@Override
public void jvmArgs(String... values) {
nodes.all(each -> each.jvmArgs(values));
}
@Override
public void jvmArgs(Supplier<String[]> valueSupplier) {
nodes.all(each -> each.jvmArgs(valueSupplier));
}
@Override
public void freeze() {
nodes.forEach(ElasticsearchNode::freeze);
@ -216,6 +226,11 @@ public class ElasticsearchCluster implements TestClusterConfiguration {
}
}
@Override
public void restart() {
nodes.forEach(ElasticsearchNode::restart);
}
@Override
public void extraConfigFile(String destination, File from) {
nodes.all(node -> node.extraConfigFile(destination, from));

79
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View File

@ -38,6 +38,7 @@ import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@ -86,6 +87,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private final Map<String, FileSupplier> keystoreFiles = new LinkedHashMap<>();
private final Map<String, Supplier<CharSequence>> systemProperties = new LinkedHashMap<>();
private final Map<String, Supplier<CharSequence>> environment = new LinkedHashMap<>();
private final List<Supplier<List<CharSequence>>> jvmArgs = new ArrayList<>();
private final Map<String, File> extraConfigFiles = new HashMap<>();
final LinkedHashMap<String, String> defaultConfig = new LinkedHashMap<>();
private final List<Map<String, String>> credentials = new ArrayList<>();
@ -105,6 +107,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
private File javaHome;
private volatile Process esProcess;
private Function<String, String> nameCustomization = Function.identity();
private boolean isWorkingDirConfigured = false;
ElasticsearchNode(String path, String name, GradleServicesAdapter services, File artifactsExtractDir, File workingDirBase) {
this.path = path;
@ -220,6 +223,19 @@ public class ElasticsearchNode implements TestClusterConfiguration {
addSupplier("Environment variable", environment, key, valueSupplier);
}
public void jvmArgs(String... values) {
for (String value : values) {
requireNonNull(value, "jvm argument was null when configuring test cluster `" + this + "`");
}
jvmArgs.add(() -> Arrays.asList(values));
}
public void jvmArgs(Supplier<String[]> valueSupplier) {
requireNonNull(valueSupplier, "jvm argument supplier was null when configuring test cluster `" + this + "`");
jvmArgs.add(() -> Arrays.asList(valueSupplier.get()));
}
private void addSupplier(String name, Map<String, Supplier<CharSequence>> collector, String key, Supplier<CharSequence> valueSupplier) {
requireNonNull(key, name + " key was null when configuring test cluster `" + this + "`");
requireNonNull(valueSupplier, name + " value supplier was null when configuring test cluster `" + this + "`");
@ -231,10 +247,13 @@ public class ElasticsearchNode implements TestClusterConfiguration {
addSupplier(name, collector, key, () -> actualValue);
}
private void checkSuppliers(String name, Map<String, Supplier<CharSequence>> collector) {
collector.forEach((key, value) -> {
requireNonNull(value.get().toString(), name + " supplied value was null when configuring test cluster `" + this + "`");
});
private void checkSuppliers(String name, Collection<Supplier<CharSequence>> collector) {
collector.forEach(suplier ->
requireNonNull(
suplier.get().toString(),
name + " supplied value was null when configuring test cluster `" + this + "`"
)
);
}
public Path getConfigDir() {
@ -289,7 +308,11 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
try {
if (isWorkingDirConfigured == false) {
// Only configure working dir once so we don't loose data on restarts
isWorkingDirConfigured = true;
createWorkingDir(distroArtifact);
}
} catch (IOException e) {
throw new UncheckedIOException("Failed to create working directory for " + this, e);
}
@ -303,7 +326,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
runElaticsearchBinScript("elasticsearch-keystore", "create");
checkSuppliers("Keystore", keystoreSettings);
checkSuppliers("Keystore", keystoreSettings.values());
keystoreSettings.forEach((key, value) ->
runElaticsearchBinScriptWithInput(value.get().toString(), "elasticsearch-keystore", "add", "-x", key)
);
@ -337,6 +360,20 @@ public class ElasticsearchNode implements TestClusterConfiguration {
startElasticsearchProcess();
}
@Override
public void restart() {
LOGGER.info("Restarting {}", this);
stop(false);
try {
Files.delete(httpPortsFile);
Files.delete(transportPortFile);
} catch (IOException e) {
throw new UncheckedIOException(e);
}
start();
}
private boolean isSettingMissingOrTrue(String name) {
return Boolean.valueOf(settings.getOrDefault(name, () -> "false").get().toString());
}
@ -349,7 +386,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
Path dst = configFile.getParent().resolve(destination);
try {
Files.createDirectories(dst);
Files.createDirectories(dst.getParent());
Files.copy(from.toPath(), dst, StandardCopyOption.REPLACE_EXISTING);
LOGGER.info("Added extra config file {} for {}", destination, this);
} catch (IOException e) {
@ -458,12 +495,30 @@ public class ElasticsearchNode implements TestClusterConfiguration {
defaultEnv.put("ES_PATH_CONF", configFile.getParent().toString());
String systemPropertiesString = "";
if (systemProperties.isEmpty() == false) {
checkSuppliers("Java System property", systemProperties);
checkSuppliers("Java System property", systemProperties.values());
systemPropertiesString = " " + systemProperties.entrySet().stream()
.map(entry -> "-D" + entry.getKey() + "=" + entry.getValue().get())
.collect(Collectors.joining(" "));
}
defaultEnv.put("ES_JAVA_OPTS", "-Xms512m -Xmx512m -ea -esa" + systemPropertiesString);
String jvmArgsString = "";
if (jvmArgs.isEmpty() == false) {
jvmArgsString = " " + jvmArgs.stream()
.map(Supplier::get)
.peek(charSequences -> requireNonNull(charSequences, "Jvm argument supplier returned null while configuring " + this))
.flatMap(Collection::stream)
.peek(argument -> {
requireNonNull(argument, "Jvm argument supplier returned null while configuring " + this);
if (argument.toString().startsWith("-D")) {
throw new TestClustersException("Invalid jvm argument `" + argument +
"` configure as systemProperty instead for " + this
);
}
})
.collect(Collectors.joining(" "));
}
defaultEnv.put("ES_JAVA_OPTS", "-Xms512m -Xmx512m -ea -esa" +
systemPropertiesString + jvmArgsString
);
defaultEnv.put("ES_TMPDIR", tmpDir.toString());
// Windows requires this as it defaults to `c:\windows` despite ES_TMPDIR
defaultEnv.put("TMP", tmpDir.toString());
@ -476,7 +531,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
);
}
checkSuppliers("Environment variable", environment);
checkSuppliers("Environment variable", environment.values());
environment.forEach((key, value) -> defaultEnv.put(key, value.get().toString()));
return defaultEnv;
}
@ -525,6 +580,10 @@ public class ElasticsearchNode implements TestClusterConfiguration {
return getTransportPortInternal();
}
public File getServerLog() {
return confPathLogs.resolve(safeName(getName()).replaceAll("-[0-9]+$", "") + "_server.json").toFile();
}
@Override
public synchronized void stop(boolean tailLogs) {
if (esProcess == null && tailLogs) {
@ -698,7 +757,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
// Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
defaultConfig.put("discovery.initial_state_timeout", "0s");
checkSuppliers("Settings", settings);
checkSuppliers("Settings", settings.values());
Map<String, String> userConfig = settings.entrySet().stream()
.collect(Collectors.toMap(entry -> entry.getKey(), entry -> entry.getValue().get().toString()));
HashSet<String> overriden = new HashSet<>(defaultConfig.keySet());

6
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java
View File

@ -66,12 +66,18 @@ public interface TestClusterConfiguration {
void environment(String key, Supplier<CharSequence> valueSupplier);
void jvmArgs(String... values);
void jvmArgs(Supplier<String[]> valueSupplier);
void freeze();
void setJavaHome(File javaHome);
void start();
void restart();
void extraConfigFile(String destination, File from);
void user(Map<String, String> userSpec);

43
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersPlugin.java
View File

@ -62,11 +62,13 @@ public class TestClustersPlugin implements Plugin<Project> {
private static final TimeUnit EXECUTOR_SHUTDOWN_TIMEOUT_UNIT = TimeUnit.MINUTES;
private static final Logger logger = Logging.getLogger(TestClustersPlugin.class);
private static final String TESTCLUSTERS_INSPECT_FAILURE = "testclusters.inspect.failure";
private final Map<Task, List<ElasticsearchCluster>> usedClusters = new HashMap<>();
private final Map<ElasticsearchCluster, Integer> claimsInventory = new HashMap<>();
private final Set<ElasticsearchCluster> runningClusters =new HashSet<>();
private final Thread shutdownHook = new Thread(this::shutDownAllClusters);
private final Boolean allowClusterToSurvive = Boolean.valueOf(System.getProperty(TESTCLUSTERS_INSPECT_FAILURE, "false"));
private ExecutorService executorService = Executors.newSingleThreadExecutor();
public static String getHelperConfigurationName(String version) {
@ -195,7 +197,7 @@ public class TestClustersPlugin implements Plugin<Project> {
public void beforeActions(Task task) {
// we only start the cluster before the actions, so we'll not start it if the task is up-to-date
usedClusters.getOrDefault(task, Collections.emptyList()).stream()
.filter(each -> runningClusters.contains(each) == false)
.filter(cluster -> runningClusters.contains(cluster) == false)
.forEach(elasticsearchCluster -> {
elasticsearchCluster.start();
runningClusters.add(elasticsearchCluster);
@ -221,18 +223,18 @@ public class TestClustersPlugin implements Plugin<Project> {
if (state.getFailure() != null) {
// If the task fails, and other tasks use this cluster, the other task will likely never be
// executed at all, so we will never get to un-claim and terminate it.
clustersUsedByTask.forEach(each -> each.stop(true));
clustersUsedByTask.forEach(cluster -> stopCluster(cluster, true));
} else {
clustersUsedByTask.forEach(
each -> claimsInventory.put(each, claimsInventory.getOrDefault(each, 0) - 1)
cluster -> claimsInventory.put(cluster, claimsInventory.getOrDefault(cluster, 0) - 1)
);
claimsInventory.entrySet().stream()
.filter(entry -> entry.getValue() == 0)
.filter(entry -> runningClusters.contains(entry.getKey()))
.map(Map.Entry::getKey)
.forEach(each -> {
each.stop(false);
runningClusters.remove(each);
.forEach(cluster -> {
stopCluster(cluster, false);
runningClusters.remove(cluster);
});
}
}
@ -242,6 +244,28 @@ public class TestClustersPlugin implements Plugin<Project> {
);
}
private void stopCluster(ElasticsearchCluster cluster, boolean taskFailed) {
if (allowClusterToSurvive) {
logger.info("Not stopping clusters, disabled by property");
if (taskFailed) {
// task failed or this is the last one to stop
for (int i=1 ; ; i += i) {
logger.lifecycle(
"No more test clusters left to run, going to sleep because {} was set," +
" interrupt (^C) to stop clusters.", TESTCLUSTERS_INSPECT_FAILURE
);
try {
Thread.sleep(1000 * i);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
return;
}
}
}
}
cluster.stop(taskFailed);
}
/**
* Boilerplate to get testClusters container extension
*
@ -428,13 +452,16 @@ public class TestClustersPlugin implements Plugin<Project> {
private void shutDownAllClusters() {
synchronized (runningClusters) {
if (runningClusters.isEmpty()) {
return;
}
Iterator<ElasticsearchCluster> iterator = runningClusters.iterator();
while (iterator.hasNext()) {
ElasticsearchCluster next = iterator.next();
iterator.remove();
iterator.next().stop(true);
next.stop(false);
}
}
}
}

1
x-pack/plugin/ccr/qa/build.gradle
View File

@ -14,4 +14,5 @@ subprojects {
include 'rest-api-spec/api/**'
}
}
}

60
x-pack/plugin/ccr/qa/downgrade-to-basic-license/build.gradle
View File

@ -1,5 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
@ -8,56 +9,51 @@ dependencies {
testCompile project(':x-pack:plugin:ccr:qa')
}
task leaderClusterTest(type: RestIntegTestTask) {
task "leader-cluster"(type: RestIntegTestTask) {
mustRunAfter(precommit)
}
leaderClusterTestCluster {
numNodes = 1
clusterName = 'leader-cluster'
setting 'xpack.license.self_generated.type', 'trial'
}
leaderClusterTestRunner {
runner {
systemProperty 'tests.target_cluster', 'leader'
}
}
testClusters."leader-cluster" {
distribution = "DEFAULT"
setting 'xpack.license.self_generated.type', 'trial'
}
task writeJavaPolicy {
ext.policyFile = file("${buildDir}/tmp/java.policy")
doLast {
final File tmp = file("${buildDir}/tmp")
if (tmp.exists() == false && tmp.mkdirs() == false) {
if (policyFile.parentFile.exists() == false && policyFile.parentFile.mkdirs() == false) {
throw new GradleException("failed to create temporary directory [${tmp}]")
}
final File javaPolicy = file("${tmp}/java.policy")
javaPolicy.write(
policyFile.write(
[
"grant {",
" permission java.io.FilePermission \"${-> followClusterTest.getNodes().get(0).homeDir}/logs/${-> followClusterTest.getNodes().get(0).clusterName}_server.json\", \"read\";",
" permission java.io.FilePermission \"${-> testClusters."follow-cluster".getFirstNode().getServerLog()}\", \"read\";",
"};"
].join("\n"))
].join("\n")
)
}
}
task followClusterTest(type: RestIntegTestTask) {}
followClusterTest.dependsOn writeJavaPolicy
task "follow-cluster"(type: RestIntegTestTask) {
dependsOn writeJavaPolicy, "leader-cluster"
useCluster testClusters."leader-cluster"
runner {
systemProperty 'java.security.policy', "file://${writeJavaPolicy.policyFile}"
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
nonInputProperties.systemProperty 'log', "${-> testClusters."follow-cluster".getFirstNode().getServerLog()}"
}
}
followClusterTestCluster {
dependsOn leaderClusterTestRunner
numNodes = 1
clusterName = 'follow-cluster'
testClusters."follow-cluster" {
distribution = "DEFAULT"
setting 'xpack.monitoring.collection.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
setting 'cluster.remote.leader_cluster.seeds', { "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\"" }
}
followClusterTestRunner {
systemProperty 'java.security.policy', "file://${buildDir}/tmp/java.policy"
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
nonInputProperties.systemProperty 'log', "${-> followClusterTest.getNodes().get(0).homeDir}/logs/" +
"${-> followClusterTest.getNodes().get(0).clusterName}_server.json"
finalizedBy 'leaderClusterTestCluster#stop'
}
check.dependsOn followClusterTest
check.dependsOn "follow-cluster"
test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test

2
x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java
View File

@ -104,7 +104,7 @@ public class FollowIndexIT extends ESCCRRestTestCase {
protected Boolean featureValueOf(JsonLogLine actual) {
return actual.level().equals("WARN") &&
actual.component().equals("o.e.x.c.a.AutoFollowCoordinator") &&
actual.nodeName().equals("node-0") &&
actual.nodeName().startsWith("follow-cluster-0") &&
actual.message().contains("failure occurred while fetching cluster state for auto follow pattern [test_pattern]") &&
actual.stacktrace().contains("org.elasticsearch.ElasticsearchStatusException: can not fetch remote cluster state " +
"as the remote cluster [leader_cluster] is not licensed for [ccr]; the license mode [BASIC]" +

77
x-pack/plugin/ccr/qa/multi-cluster/build.gradle
View File

@ -1,5 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
@ -8,57 +9,57 @@ dependencies {
testCompile project(':x-pack:plugin:ccr:qa')
}
task leaderClusterTest(type: RestIntegTestTask) {
task "leader-cluster"(type: RestIntegTestTask) {
mustRunAfter(precommit)
}
leaderClusterTestCluster {
numNodes = 1
clusterName = 'leader-cluster'
setting 'xpack.license.self_generated.type', 'trial'
setting 'node.name', 'leader'
}
leaderClusterTestRunner {
runner {
systemProperty 'tests.target_cluster', 'leader'
}
}
task middleClusterTest(type: RestIntegTestTask) {}
middleClusterTestCluster {
dependsOn leaderClusterTestRunner
numNodes = 1
clusterName = 'middle-cluster'
testClusters."leader-cluster" {
distribution = "DEFAULT"
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
setting 'node.name', 'middle'
}
middleClusterTestRunner {
task "middle-cluster"(type: RestIntegTestTask) {
dependsOn "leader-cluster"
useCluster testClusters."leader-cluster"
runner {
systemProperty 'tests.target_cluster', 'middle'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
nonInputProperties.systemProperty 'tests.leader_host',
"${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
}
}
testClusters."middle-cluster" {
distribution = "DEFAULT"
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\"" }
}
task followClusterTest(type: RestIntegTestTask) {}
task 'follow-cluster'(type: RestIntegTestTask) {
dependsOn "leader-cluster", "middle-cluster"
useCluster testClusters."leader-cluster"
useCluster testClusters."middle-cluster"
runner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host',
"${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
nonInputProperties.systemProperty 'tests.middle_host',
"${-> testClusters."middle-cluster".getAllHttpSocketURI().get(0)}"
}
}
followClusterTestCluster {
dependsOn middleClusterTestRunner
numNodes = 1
clusterName = 'follow-cluster'
testClusters."follow-cluster" {
distribution = "DEFAULT"
setting 'xpack.monitoring.collection.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
setting 'cluster.remote.middle_cluster.seeds', "\"${-> middleClusterTest.nodes.get(0).transportUri()}\""
setting 'node.name', 'follow'
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${testClusters."leader-cluster".getAllTransportPortURI().join(",")}\""}
setting 'cluster.remote.middle_cluster.seeds',
{ "\"${testClusters."middle-cluster".getAllTransportPortURI().join(",")}\""}
}
followClusterTestRunner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
nonInputProperties.systemProperty 'tests.middle_host', "${-> middleClusterTest.nodes.get(0).httpUri()}"
finalizedBy 'leaderClusterTestCluster#stop'
finalizedBy 'middleClusterTestCluster#stop'
}
check.dependsOn followClusterTest
check.dependsOn "follow-cluster"
test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test

42
x-pack/plugin/ccr/qa/non-compliant-license/build.gradle
View File

@ -1,5 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
@ -8,34 +9,31 @@ dependencies {
testCompile project(':x-pack:plugin:ccr:qa:')
}
task leaderClusterTest(type: RestIntegTestTask) {
task 'leader-cluster'(type: RestIntegTestTask) {
mustRunAfter(precommit)
}
leaderClusterTestCluster {
numNodes = 1
clusterName = 'leader-cluster'
}
leaderClusterTestRunner {
runner {
systemProperty 'tests.target_cluster', 'leader'
}
task followClusterTest(type: RestIntegTestTask) {}
followClusterTestCluster {
dependsOn leaderClusterTestRunner
numNodes = 1
clusterName = 'follow-cluster'
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
}
testClusters.'leader-cluster' {
distribution = "DEFAULT"
}
followClusterTestRunner {
task 'follow-cluster'(type: RestIntegTestTask) {
dependsOn 'leader-cluster'
useCluster testClusters.'leader-cluster'
runner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
finalizedBy 'leaderClusterTestCluster#stop'
nonInputProperties.systemProperty 'tests.leader_host',
{ "${testClusters.'follow-cluster'.getAllHttpSocketURI().get(0)}" }
}
}
testClusters.'follow-cluster' {
distribution = "DEFAULT"
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${testClusters.'leader-cluster'.getAllTransportPortURI().join(",")}\"" }
}
check.dependsOn followClusterTest
check.dependsOn "follow-cluster"
test.enabled = false

20
x-pack/plugin/ccr/qa/rest/build.gradle
View File

@ -1,5 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
@ -11,29 +12,18 @@ task restTest(type: RestIntegTestTask) {
mustRunAfter(precommit)
}
restTestCluster {
distribution 'default'
testClusters.restTest {
distribution = 'default'
// Disable assertions in FollowingEngineAssertions, otherwise an AssertionError is thrown before
// indexing a document directly in a follower index. In a rest test we like to test the exception
// that is thrown in production when indexing a document directly in a follower index.
environment 'ES_JAVA_OPTS', '-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions'
jvmArgs '-da:org.elasticsearch.xpack.ccr.index.engine.FollowingEngineAssertions'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.security.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
// TODO: reduce the need for superuser here
setupCommand 'setup-ccr-user',
'bin/elasticsearch-users', 'useradd', 'ccr-user', '-p', 'ccr-user-password', '-r', 'superuser'
waitCondition = { node, ant ->
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
dest: tmpFile.toString(),
username: 'ccr-user',
password: 'ccr-user-password',
ignoreerrors: true,
retries: 10)
return tmpFile.exists()
}
user username:'ccr-user', password: 'ccr-user-password', role: 'superuser'
}
check.dependsOn restTest

67
x-pack/plugin/ccr/qa/restart/build.gradle
View File

@ -1,60 +1,55 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
testCompile project(':x-pack:plugin:ccr:qa')
}
task leaderClusterTest(type: RestIntegTestTask) {
task 'leader-cluster'(type: RestIntegTestTask) {
mustRunAfter(precommit)
}
leaderClusterTestCluster {
numNodes = 1
clusterName = 'leader-cluster'
setting 'xpack.license.self_generated.type', 'trial'
setting 'node.name', 'leader'
}
leaderClusterTestRunner {
runner {
systemProperty 'tests.target_cluster', 'leader'
}
task followClusterTest(type: RestIntegTestTask) {}
followClusterTestCluster {
dependsOn leaderClusterTestRunner
numNodes = 1
clusterName = 'follow-cluster'
setting 'xpack.monitoring.collection.enabled', 'true'
}
testClusters.'leader-cluster' {
distribution = "DEFAULT"
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
setting 'node.name', 'follow'
}
followClusterTestRunner {
task 'follow-cluster'(type: RestIntegTestTask) {
dependsOn 'leader-cluster'
useCluster testClusters.'leader-cluster'
runner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
nonInputProperties.systemProperty 'tests.leader_host',
"${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}"
}
task followClusterRestartTest(type: RestIntegTestTask) {}
followClusterRestartTestCluster {
dependsOn followClusterTestRunner, 'followClusterTestCluster#stop'
numNodes = 1
clusterName = 'follow-cluster'
dataDir = { nodeNumber -> followClusterTest.nodes[0].dataDir }
}
testClusters.'follow-cluster' {
distribution = "DEFAULT"
setting 'xpack.monitoring.collection.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
setting 'node.name', 'follow'
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" }
nameCustomization = { 'follow' }
}
followClusterRestartTestRunner {
task followClusterRestartTest(type: Test) {
dependsOn tasks.'follow-cluster'
useCluster testClusters.'leader-cluster'
useCluster testClusters.'follow-cluster'
maxParallelForks = 1
systemProperty 'tests.rest.load_packaged', 'false'
systemProperty 'tests.target_cluster', 'follow-restart'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
finalizedBy 'leaderClusterTestCluster#stop'
doFirst {
testClusters.'follow-cluster'.restart()
nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}"
nonInputProperties.systemProperty 'tests.rest.cluster', "${-> testClusters.'follow-cluster'.getAllHttpSocketURI().join(",")}"
}
outputs.doNotCacheIf "Caching of REST tests not implemented yet", { false }
}
check.dependsOn followClusterRestartTest

76
x-pack/plugin/ccr/qa/security/build.gradle
View File

@ -1,5 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
@ -8,69 +9,44 @@ dependencies {
testCompile project(':x-pack:plugin:ccr:qa')
}
task leaderClusterTest(type: RestIntegTestTask) {
task 'leader-cluster'(type: RestIntegTestTask) {
mustRunAfter(precommit)
runner {
systemProperty 'tests.target_cluster', 'leader'
}
}
leaderClusterTestCluster {
numNodes = 1
clusterName = 'leader-cluster'
testClusters.'leader-cluster' {
distribution = 'Default'
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
setting 'xpack.monitoring.enabled', 'false'
extraConfigFile 'roles.yml', 'leader-roles.yml'
setupCommand 'setupTestAdmin',
'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
setupCommand 'setupCcrUser',
'bin/elasticsearch-users', 'useradd', "test_ccr", '-p', 'x-pack-test-password', '-r', "ccruser"
waitCondition = { node, ant ->
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
dest: tmpFile.toString(),
username: 'test_admin',
password: 'x-pack-test-password',
ignoreerrors: true,
retries: 10)
return tmpFile.exists()
extraConfigFile 'roles.yml', file('leader-roles.yml')
user username: "test_admin", role: "superuser"
user username: "test_ccr", role: "ccruser"
}
task 'follow-cluster'(type: RestIntegTestTask) {
dependsOn 'leader-cluster'
useCluster testClusters.'leader-cluster'
runner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> testClusters.'leader-cluster'.getAllHttpSocketURI().get(0)}"
}
}
leaderClusterTestRunner {
systemProperty 'tests.target_cluster', 'leader'
testClusters.'follow-cluster' {
distribution = 'Default'
setting 'cluster.remote.leader_cluster.seeds', {
"\"${testClusters.'leader-cluster'.getAllTransportPortURI().join(",")}\""
}
task followClusterTest(type: RestIntegTestTask) {}
followClusterTestCluster {
dependsOn leaderClusterTestRunner
numNodes = 1
clusterName = 'follow-cluster'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
setting 'xpack.license.self_generated.type', 'trial'
setting 'xpack.security.enabled', 'true'
setting 'xpack.monitoring.collection.enabled', 'true'
extraConfigFile 'roles.yml', 'follower-roles.yml'
setupCommand 'setupTestAdmin',
'bin/elasticsearch-users', 'useradd', "test_admin", '-p', 'x-pack-test-password', '-r', "superuser"
setupCommand 'setupCcrUser',
'bin/elasticsearch-users', 'useradd', "test_ccr", '-p', 'x-pack-test-password', '-r', "ccruser"
waitCondition = { node, ant ->
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
dest: tmpFile.toString(),
username: 'test_admin',
password: 'x-pack-test-password',
ignoreerrors: true,
retries: 10)
return tmpFile.exists()
}
extraConfigFile 'roles.yml', file('follower-roles.yml')
user username: "test_admin", role: "superuser"
user username: "test_ccr", role: "ccruser"
}
followClusterTestRunner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
finalizedBy 'leaderClusterTestCluster#stop'
}
check.dependsOn followClusterTest
check.dependsOn 'follow-cluster'
test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test