honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Convert ILM tests to use testclusters (#43076) 

Also improove the error message when bin scripts are not found
This commit is contained in:
Alpar Torok 2019-06-13 12:18:55 +03:00
parent 209a493b27
commit 167e51335d
7 changed files with 80 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View File

@ -409,6 +409,13 @@ public class ElasticsearchNode implements TestClusterConfiguration {
}
private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
if (
Files.exists(workingDir.resolve("bin").resolve(tool)) == false &&
Files.exists(workingDir.resolve("bin").resolve(tool + ".bat")) == false
) {
throw new TestClustersException("Can't run bin script: `" + tool + "` does not exist. " +
"Is this the distribution you expect it to be ?");
}
try (InputStream byteArrayInputStream = new ByteArrayInputStream(input.getBytes(StandardCharsets.UTF_8))) {
services.loggedExec(spec -> {
spec.setEnvironment(getESEnvironment());

55
x-pack/plugin/ilm/qa/multi-cluster/build.gradle
View File

@ -1,5 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
@ -8,13 +9,20 @@ dependencies {
testCompile project(':x-pack:plugin:ilm')
}
task leaderClusterTest(type: RestIntegTestTask) {
File repoDir = file("$buildDir/testclusters/repo")
task 'leader-cluster'(type: RestIntegTestTask) {
mustRunAfter(precommit)
runner {
systemProperty 'tests.target_cluster', 'leader'
/* To support taking index snapshots, we have to set path.repo setting */
systemProperty 'tests.path.repo', repoDir.absolutePath
}
}
leaderClusterTestCluster {
numNodes = 1
clusterName = 'leader-cluster'
testClusters.'leader-cluster' {
distribution = "DEFAULT"
setting 'path.repo', repoDir.absolutePath
setting 'xpack.ilm.enabled', 'true'
setting 'xpack.ccr.enabled', 'true'
setting 'xpack.security.enabled', 'false'
@ -25,18 +33,23 @@ leaderClusterTestCluster {
setting 'indices.lifecycle.poll_interval', '1000ms'
}
leaderClusterTestRunner {
systemProperty 'tests.target_cluster', 'leader'
/* To support taking index snapshots, we have to set path.repo setting */
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
task 'follow-cluster'(type: RestIntegTestTask) {
dependsOn 'leader-cluster'
useCluster testClusters.'leader-cluster'
runner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host',
"${-> testClusters."leader-cluster".getAllHttpSocketURI().get(0)}"
nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed',
"${-> testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}"
/* To support taking index snapshots, we have to set path.repo setting */
systemProperty 'tests.path.repo', repoDir.absolutePath
}
}
task followClusterTest(type: RestIntegTestTask) {}
followClusterTestCluster {
dependsOn leaderClusterTestRunner
numNodes = 1
clusterName = 'follow-cluster'
testClusters.'follow-cluster' {
distribution = "DEFAULT"
setting 'path.repo', repoDir.absolutePath
setting 'xpack.ilm.enabled', 'true'
setting 'xpack.ccr.enabled', 'true'
setting 'xpack.security.enabled', 'false'
@ -45,17 +58,9 @@ followClusterTestCluster {
setting 'xpack.ml.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setting 'indices.lifecycle.poll_interval', '1000ms'
setting 'cluster.remote.leader_cluster.seeds', "\"${-> leaderClusterTest.nodes.get(0).transportUri()}\""
setting 'cluster.remote.leader_cluster.seeds',
{ "\"${testClusters.'leader-cluster'.getAllTransportPortURI().get(0)}\"" }
}
followClusterTestRunner {
systemProperty 'tests.target_cluster', 'follow'
nonInputProperties.systemProperty 'tests.leader_host', "${-> leaderClusterTest.nodes.get(0).httpUri()}"
nonInputProperties.systemProperty 'tests.leader_remote_cluster_seed', "${-> leaderClusterTest.nodes.get(0).transportUri()}"
/* To support taking index snapshots, we have to set path.repo setting */
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
finalizedBy 'leaderClusterTestCluster#stop'
}
check.dependsOn followClusterTest
check.dependsOn 'follow-cluster'
test.enabled = false // no unit tests for this module, only the rest integration test

16
x-pack/plugin/ilm/qa/multi-node/build.gradle
View File

@ -1,3 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
@ -5,15 +8,18 @@ dependencies {
testCompile project(path: xpackProject('plugin').path, configuration: 'testArtifacts')
}
integTestRunner {
File repoDir = file("$buildDir/testclusters/repo")
integTest.runner {
/* To support taking index snapshots, we have to set path.repo setting */
systemProperty 'tests.path.repo', new File(buildDir, "cluster/shared/repo")
systemProperty 'tests.path.repo', repoDir
}
integTestCluster {
numNodes = 4
clusterName = 'ilm'
testClusters.integTest {
distribution = 'DEFAULT'
numberOfNodes = 4
setting 'path.repo', repoDir.absolutePath
setting 'xpack.ilm.enabled', 'true'
setting 'xpack.security.enabled', 'false'
setting 'xpack.watcher.enabled', 'false'

8
x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/ChangePolicyforIndexIT.java
View File

@ -63,7 +63,7 @@ public class ChangePolicyforIndexIT extends ESRestTestCase {
Map<String, Phase> phases2 = new HashMap<>();
phases2.put("hot", new Phase("hot", TimeValue.ZERO, singletonMap(RolloverAction.NAME, new RolloverAction(null, null, 1000L))));
phases2.put("warm", new Phase("warm", TimeValue.ZERO,
singletonMap(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "node-1,node-2"), null, null))));
singletonMap(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "integTest-1,integTest-2"), null, null))));
LifecyclePolicy lifecyclePolicy2 = new LifecyclePolicy("policy_1", phases2);
// PUT policy_1 and policy_2
XContentBuilder builder1 = jsonBuilder();
@ -81,7 +81,7 @@ public class ChangePolicyforIndexIT extends ESRestTestCase {
// create the test-index index and set the policy to policy_1
Settings settings = Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put("index.routing.allocation.include._name", "node-0")
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put("index.routing.allocation.include._name", "integTest-0")
.put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias").put(LifecycleSettings.LIFECYCLE_NAME, "policy_1").build();
Request createIndexRequest = new Request("PUT", "/" + indexName);
createIndexRequest.setJsonEntity(
@ -114,7 +114,7 @@ public class ChangePolicyforIndexIT extends ESRestTestCase {
// Check the index goes to the warm phase and completes
assertBusy(() -> assertStep(indexName, TerminalPolicyStep.KEY));
// Check index is allocated on node-1 and node-2 as per policy_2
// Check index is allocated on integTest-1 and integTest-2 as per policy_2
Request getSettingsRequest = new Request("GET", "/" + indexName + "/_settings");
Response getSettingsResponse = client().performRequest(getSettingsRequest);
assertOK(getSettingsResponse);
@ -127,7 +127,7 @@ public class ChangePolicyforIndexIT extends ESRestTestCase {
@SuppressWarnings("unchecked")
String includesAllocation = (String) ((Map<String, Object>) ((Map<String, Object>) routingSettings.get("allocation"))
.get("include")).get("_name");
assertEquals("node-1,node-2", includesAllocation);
assertEquals("integTest-1,integTest-2", includesAllocation);
}
private void assertStep(String indexName, StepKey expectedStep) throws IOException {

14
x-pack/plugin/ilm/qa/multi-node/src/test/java/org/elasticsearch/xpack/indexlifecycle/TimeSeriesLifecycleActionsIT.java
View File

@ -86,7 +86,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
String secondIndex = index + "-000002";
createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put("index.routing.allocation.include._name", "node-0")
.put("index.routing.allocation.include._name", "integTest-0")
.put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
// create policy
@ -114,7 +114,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
String originalIndex = index + "-000001";
createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put("index.routing.allocation.include._name", "node-0")
.put("index.routing.allocation.include._name", "integTest-0")
.put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
// create policy
@ -148,7 +148,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
String secondIndex = index + "-000002";
createIndexWithSettings(originalIndex, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 4)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
.put("index.routing.allocation.include._name", "node-0")
.put("index.routing.allocation.include._name", "integTest-0")
.put(RolloverAction.LIFECYCLE_ROLLOVER_ALIAS, "alias"));
createFullPolicy(TimeValue.timeValueHours(10));
@ -313,7 +313,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
public void testAllocateOnlyAllocation() throws Exception {
createIndexWithSettings(index, Settings.builder().put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0));
String allocateNodeName = "node-" + randomFrom(0, 1);
String allocateNodeName = "integTest-" + randomFrom(0, 1);
AllocateAction allocateAction = new AllocateAction(null, null, null, singletonMap("_name", allocateNodeName));
createNewSingletonPolicy(randomFrom("warm", "cold"), allocateAction);
updatePolicy(index, policy);
@ -509,7 +509,7 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 2)
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0)
// required so the shrink doesn't wait on SetSingleNodeAllocateStep
.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "node-0"));
.put(IndexMetaData.INDEX_ROUTING_REQUIRE_GROUP_SETTING.getKey() + "_name", "integTest-0"));
// index document so snapshot actually does something
indexDocument();
// start snapshot
@ -846,11 +846,11 @@ public class TimeSeriesLifecycleActionsIT extends ESRestTestCase {
Map<String, LifecycleAction> warmActions = new HashMap<>();
warmActions.put(SetPriorityAction.NAME, new SetPriorityAction(50));
warmActions.put(ForceMergeAction.NAME, new ForceMergeAction(1));
warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "node-1,node-2"), null, null));
warmActions.put(AllocateAction.NAME, new AllocateAction(1, singletonMap("_name", "integTest-1,integTest-2"), null, null));
warmActions.put(ShrinkAction.NAME, new ShrinkAction(1));
Map<String, LifecycleAction> coldActions = new HashMap<>();
coldActions.put(SetPriorityAction.NAME, new SetPriorityAction(0));
coldActions.put(AllocateAction.NAME, new AllocateAction(0, singletonMap("_name", "node-3"), null, null));
coldActions.put(AllocateAction.NAME, new AllocateAction(0, singletonMap("_name", "integTest-3"), null, null));
Map<String, Phase> phases = new HashMap<>();
phases.put("hot", new Phase("hot", hotTime, hotActions));
phases.put("warm", new Phase("warm", TimeValue.ZERO, warmActions));

32
x-pack/plugin/ilm/qa/rest/build.gradle
View File

@ -1,5 +1,6 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
@ -7,38 +8,25 @@ dependencies {
testCompile project(path: xpackModule('ilm'), configuration: 'runtime')
}
task restTest(type: RestIntegTestTask) {
mustRunAfter(precommit)
}
def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
restTestRunner {
systemProperty 'tests.rest.cluster.username', clusterCredentials.username
systemProperty 'tests.rest.cluster.password', clusterCredentials.password
task restTest(type: RestIntegTestTask) {
mustRunAfter(precommit)
runner {
systemProperty 'tests.rest.cluster.username', clusterCredentials.username
systemProperty 'tests.rest.cluster.password', clusterCredentials.password
}
}
restTestCluster {
distribution 'default'
testClusters.restTest {
distribution = 'DEFAULT'
setting 'xpack.ilm.enabled', 'true'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.security.enabled', 'true'
setting 'xpack.license.self_generated.type', 'trial'
setupCommand 'setup-admin-user',
'bin/elasticsearch-users', 'useradd', clusterCredentials.username, '-p', clusterCredentials.password, '-r', 'superuser'
waitCondition = { node, ant ->
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
dest: tmpFile.toString(),
username: clusterCredentials.username,
password: clusterCredentials.password,
ignoreerrors: true,
retries: 10)
return tmpFile.exists()
}
user clusterCredentials
}
check.dependsOn restTest

35
x-pack/plugin/ilm/qa/with-security/build.gradle
View File

@ -1,3 +1,4 @@
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
@ -8,36 +9,22 @@ dependencies {
def clusterCredentials = [username: System.getProperty('tests.rest.cluster.username', 'test_admin'),
password: System.getProperty('tests.rest.cluster.password', 'x-pack-test-password')]
integTestRunner {
systemProperty 'tests.rest.cluster.username', clusterCredentials.username
systemProperty 'tests.rest.cluster.password', clusterCredentials.password
integTest {
runner {
systemProperty 'tests.rest.cluster.username', clusterCredentials.username
systemProperty 'tests.rest.cluster.password', clusterCredentials.password
}
}
integTestCluster {
testClusters.integTest {
distribution = "DEFAULT"
setting 'xpack.ilm.enabled', 'true'
setting 'xpack.security.enabled', 'true'
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.ml.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
extraConfigFile 'roles.yml', 'roles.yml'
setupCommand 'setupIlmUser',
'bin/elasticsearch-users',
'useradd', "test_ilm",
'-p', 'x-pack-test-password', '-r', "ilm"
setupCommand 'setupDummyUser',
'bin/elasticsearch-users',
'useradd', clusterCredentials.username,
'-p', clusterCredentials.password,
'-r', 'superuser'
waitCondition = { node, ant ->
File tmpFile = new File(node.cwd, 'wait.success')
ant.get(src: "http://${node.httpUri()}/_cluster/health?wait_for_nodes=>=${numNodes}&wait_for_status=yellow",
dest: tmpFile.toString(),
username: clusterCredentials.username,
password: clusterCredentials.password,
ignoreerrors: true,
retries: 10)
return tmpFile.exists()
}
extraConfigFile 'roles.yml', file('roles.yml')
user clusterCredentials
user username: "test_ilm", password: "x-pack-test-password", role: "ilm"
}