honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[Transform] Add yml test suite for testing remote clusters (CCS) (#51033) 

add a test suite for remote clusters features and add test cases for transform
This commit is contained in:
Hendrik Muhs 2020-01-22 11:17:59 +01:00
parent 86a50a24f3
commit af76ae4ab9
4 changed files with 496 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
x-pack/qa/multi-cluster-tests-with-security/build.gradle Normal file
View File

@ -0,0 +1,59 @@
import org.elasticsearch.gradle.test.RestIntegTestTask
apply plugin: 'elasticsearch.testclusters'
apply plugin: 'elasticsearch.standalone-test'
dependencies {
testCompile project(':x-pack:qa')
testCompile project(':client:rest-high-level')
}
task 'remote-cluster'(type: RestIntegTestTask) {
mustRunAfter(precommit)
runner {
systemProperty 'tests.rest.suite', 'remote_cluster'
}
}
testClusters.'remote-cluster' {
testDistribution = 'DEFAULT'
numberOfNodes = 2
setting 'cluster.remote.connect', "false"
setting 'xpack.ilm.enabled', 'false'
setting 'xpack.security.enabled', 'true'
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
user username: "test_user", password: "x-pack-test-password"
}
task 'mixed-cluster'(type: RestIntegTestTask) {
dependsOn 'remote-cluster'
runner {
useCluster testClusters.'remote-cluster'
systemProperty 'tests.rest.suite', 'multi_cluster'
}
}
testClusters.'mixed-cluster' {
testDistribution = 'DEFAULT'
setting 'xpack.security.enabled', 'true'
setting 'xpack.watcher.enabled', 'false'
setting 'xpack.monitoring.enabled', 'false'
setting 'xpack.license.self_generated.type', 'trial'
setting 'cluster.remote.my_remote_cluster.seeds', {
testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString()
}
setting 'cluster.remote.connections_per_cluster', "1"
setting 'cluster.remote.connect', "true"
user username: "test_user", password: "x-pack-test-password"
}
task integTest {
dependsOn 'mixed-cluster'
}
test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test
check.dependsOn(integTest)

50
x-pack/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java Normal file
View File

@ -0,0 +1,50 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License;
* you may not use this file except in compliance with the Elastic License.
*/
package org.elasticsearch.multi_cluster;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite;
import org.apache.lucene.util.TimeUnits;
import org.elasticsearch.common.settings.SecureString;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.concurrent.ThreadContext;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue;
@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs
public class MultiClusterYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
private static final String USER = "test_user";
private static final String PASS = "x-pack-test-password";
@Override
protected boolean preserveIndicesUponCompletion() {
return true;
}
public MultiClusterYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return createParameters();
}
@Override
protected Settings restClientSettings() {
String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray()));
return Settings.builder()
.put(super.restClientSettings())
.put(ThreadContext.PREFIX + ".Authorization", token)
.build();
}
}

261
x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml Normal file
View File

@ -0,0 +1,261 @@
---
setup:
- skip:
features: headers
- do:
cluster.health:
wait_for_status: yellow
- do:
security.put_user:
username: "joe"
body: >
{
"password": "transform",
"roles" : [ "transform_admin", "x_cluster_role" ]
}
- do:
security.put_role:
name: "x_cluster_role"
body: >
{
"cluster": [],
"indices": [
{
"names": ["test_index", "my_remote_cluster:test_i*", "my_remote_cluster:aliased_test_index"],
"privileges": ["all", "view_index_metadata"]
},
{
"names": ["simple-remote-transform", "simple-local-remote-transform"],
"privileges": ["all"]
}
]
}
---
teardown:
- do:
security.delete_user:
username: "joe"
ignore: 404
---
"Search remote cluster":
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
search:
rest_total_hits_as_int: true
index: my_remote_cluster:test_index
body:
aggs:
user:
terms:
field: user
- match: { _shards.total: 3 }
- match: { hits.total: 9 }
- length: { aggregations.user.buckets: 3 }
- match: { aggregations.user.buckets.0.key: "a" }
- match: { aggregations.user.buckets.0.doc_count: 5 }
---
"Batch transform from remote cluster":
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.put_transform:
transform_id: "simple-remote-transform"
body: >
{
"source": { "index": "my_remote_cluster:test_index" },
"dest": { "index": "simple-remote-transform" },
"pivot": {
"group_by": { "user": {"terms": {"field": "user"}}},
"aggs": {"avg_stars": {"avg": {"field": "stars"}}}
}
}
- match: { acknowledged: true }
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.start_transform:
transform_id: "simple-remote-transform"
- match: { acknowledged: true }
- do:
transform.get_transform_stats:
transform_id: "simple-remote-transform"
- match: { count: 1 }
- match: { transforms.0.id: "simple-remote-transform" }
- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.stop_transform:
transform_id: "simple-remote-transform"
wait_for_completion: true
wait_for_checkpoint: true
- match: { acknowledged: true }
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.get_transform_stats:
transform_id: "simple-remote-transform"
- match: { count: 1 }
- match: { transforms.0.id: "simple-remote-transform" }
- match: { transforms.0.state: "stopped" }
- match: { transforms.0.checkpointing.last.checkpoint: 1 }
# workaround: refresh dest index, to be removed, see gh #51154
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
indices.refresh:
index: simple-remote-transform
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
search:
rest_total_hits_as_int: true
index: simple-remote-transform
sort: user
- match: { hits.total: 3 }
- match: { hits.hits.0._index: simple-remote-transform }
- match: { hits.hits.0._source.avg_stars: 3.6 }
- match: { hits.hits.0._source.user: a }
- match: { hits.hits.1._source.avg_stars: 2.0 }
- match: { hits.hits.1._source.user: b }
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.update_transform:
transform_id: "simple-remote-transform"
body: >
{
"source": { "index": ["my_remote_cluster:test_index", "my_remote_cluster:test_index_2"] }
}
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.get_transform_stats:
transform_id: "simple-remote-transform"
- match: { count: 1 }
- match: { transforms.0.id: "simple-remote-transform" }
- match: { transforms.0.state: "stopped" }
# we added test_index_2, which has 2 more docs:
- match: { transforms.0.checkpointing.operations_behind: 2 }
---
"Batch transform from local and remote cluster":
- do:
indices.create:
index: test_index
body:
settings:
index:
number_of_shards: 3
number_of_replicas: 0
aliases:
test_alias: {}
mappings:
properties:
time:
type: date
user:
type: keyword
stars:
type: integer
coolness:
type: integer
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_index"}}'
- '{"user": "a", "stars": 3, "date" : "2018-11-29T12:12:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "c", "stars": 5, "date" : "2018-11-29T12:14:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "d", "stars": 5, "date" : "2018-11-29T12:16:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "e", "stars": 2, "date" : "2018-11-29T12:17:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "b", "stars": 3, "date" : "2018-11-29T12:22:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "c", "stars": 5, "date" : "2018-11-29T12:23:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "d", "stars": 1, "date" : "2018-11-29T12:32:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "e", "stars": 3, "date" : "2018-11-29T12:34:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "c", "stars": 4, "date" : "2018-11-29T12:35:12.123456789Z"}'
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.put_transform:
transform_id: "simple-local-remote-transform"
body: >
{
"source": { "index": ["test_index", "my_remote_cluster:test_index"] },
"dest": { "index": "simple-local-remote-transform" },
"pivot": {
"group_by": { "user": {"terms": {"field": "user"}}},
"aggs": {
"avg_stars": {"avg": {"field": "stars"}},
"count": {"value_count": {"field": "user"}}
}
}
}
- match: { acknowledged: true }
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.start_transform:
transform_id: "simple-local-remote-transform"
- match: { acknowledged: true }
- do:
transform.get_transform_stats:
transform_id: "simple-local-remote-transform"
- match: { count: 1 }
- match: { transforms.0.id: "simple-local-remote-transform" }
- match: { transforms.0.state: "/started|indexing|stopping|stopped/" }
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.stop_transform:
transform_id: "simple-local-remote-transform"
wait_for_completion: true
wait_for_checkpoint: true
- match: { acknowledged: true }
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
transform.get_transform_stats:
transform_id: "simple-local-remote-transform"
- match: { count: 1 }
- match: { transforms.0.id: "simple-local-remote-transform" }
- match: { transforms.0.state: "stopped" }
- match: { transforms.0.checkpointing.last.checkpoint: 1 }
# workaround: refresh dest index, to be removed, see gh #51154
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
indices.refresh:
index: simple-local-remote-transform
- do:
headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" }
search:
rest_total_hits_as_int: true
index: simple-local-remote-transform
sort: user
- match: { hits.total: 5 }
- match: { hits.hits.0._index: simple-local-remote-transform }
- match: { hits.hits.0._source.avg_stars: 3.5 }
- match: { hits.hits.0._source.count: 6 }
- match: { hits.hits.0._source.user: a }
- match: { hits.hits.1._source.avg_stars: 2.25 }
- match: { hits.hits.1._source.user: b }
- match: { hits.hits.3._source.count: 2 }
- match: { hits.hits.3._source.user: d }

126
x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml Normal file
View File

@ -0,0 +1,126 @@
---
setup:
- skip:
features: headers
- do:
cluster.health:
wait_for_status: yellow
- do:
security.put_user:
username: "joe"
body: >
{
"password": "transform",
"roles" : [ "x_cluster_role" ]
}
- do:
security.put_role:
name: "x_cluster_role"
body: >
{
"cluster": [],
"indices": [
{
"names": ["test_index*"],
"privileges": ["read", "view_index_metadata"]
}
]
}
---
teardown:
- do:
security.delete_user:
username: "joe"
ignore: 404
---
"Index data on the remote cluster":
- do:
indices.create:
index: test_index
body:
settings:
index:
number_of_shards: 3
number_of_replicas: 0
aliases:
test_alias: {}
mappings:
properties:
time:
type: date
user:
type: keyword
stars:
type: integer
coolness:
type: integer
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_index"}}'
- '{"user": "a", "stars": 1, "date" : "2018-10-29T12:12:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "a", "stars": 4, "date" : "2018-10-29T12:14:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "a", "stars": 5, "date" : "2018-10-29T12:16:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "b", "stars": 2, "date" : "2018-10-29T12:17:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "b", "stars": 3, "date" : "2018-10-29T12:22:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "a", "stars": 5, "date" : "2018-10-29T12:23:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "b", "stars": 1, "date" : "2018-10-29T12:32:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "a", "stars": 3, "date" : "2018-10-29T12:34:12.123456789Z"}'
- '{"index": {"_index": "test_index"}}'
- '{"user": "c", "stars": 4, "date" : "2018-10-29T12:35:12.123456789Z"}'
- do:
search:
rest_total_hits_as_int: true
index: test_index
body:
aggs:
user:
terms:
field: user
- match: { _shards.total: 3 }
- match: { hits.total: 9 }
- length: { aggregations.user.buckets: 3 }
- match: { aggregations.user.buckets.0.key: "a" }
- match: { aggregations.user.buckets.0.doc_count: 5 }
- do:
indices.create:
index: test_index_2
body:
settings:
index:
number_of_shards: 3
number_of_replicas: 0
aliases:
test_alias: {}
mappings:
properties:
time:
type: date
user:
type: keyword
stars:
type: integer
coolness:
type: integer
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test_index_2"}}'
- '{"user": "e", "stars": 3, "date" : "2018-10-29T12:12:12.123456789Z"}'
- '{"index": {"_index": "test_index_2"}}'
- '{"user": "d", "stars": 4, "date" : "2018-10-29T12:14:12.123456789Z"}'