From af76ae4ab929a1b3c84a484fb92f0419013e00d3 Mon Sep 17 00:00:00 2001 From: Hendrik Muhs Date: Wed, 22 Jan 2020 11:17:59 +0100 Subject: [PATCH] [Transform] Add yml test suite for testing remote clusters (CCS) (#51033) add a test suite for remote clusters features and add test cases for transform --- .../build.gradle | 59 ++++ .../MultiClusterYamlTestSuiteIT.java | 50 ++++ .../test/multi_cluster/80_transform.yml | 261 ++++++++++++++++++ .../test/remote_cluster/80_transform.yml | 126 +++++++++ 4 files changed, 496 insertions(+) create mode 100644 x-pack/qa/multi-cluster-tests-with-security/build.gradle create mode 100644 x-pack/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java create mode 100644 x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml create mode 100644 x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml diff --git a/x-pack/qa/multi-cluster-tests-with-security/build.gradle b/x-pack/qa/multi-cluster-tests-with-security/build.gradle new file mode 100644 index 00000000000..74b15cd87a5 --- /dev/null +++ b/x-pack/qa/multi-cluster-tests-with-security/build.gradle @@ -0,0 +1,59 @@ +import org.elasticsearch.gradle.test.RestIntegTestTask + +apply plugin: 'elasticsearch.testclusters' +apply plugin: 'elasticsearch.standalone-test' + +dependencies { + testCompile project(':x-pack:qa') + testCompile project(':client:rest-high-level') +} + +task 'remote-cluster'(type: RestIntegTestTask) { + mustRunAfter(precommit) + runner { + systemProperty 'tests.rest.suite', 'remote_cluster' + } +} + +testClusters.'remote-cluster' { + testDistribution = 'DEFAULT' + numberOfNodes = 2 + setting 'cluster.remote.connect', "false" + setting 'xpack.ilm.enabled', 'false' + setting 'xpack.security.enabled', 'true' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + + user username: "test_user", password: "x-pack-test-password" +} + +task 'mixed-cluster'(type: RestIntegTestTask) { + dependsOn 'remote-cluster' + runner { + useCluster testClusters.'remote-cluster' + systemProperty 'tests.rest.suite', 'multi_cluster' + } +} + +testClusters.'mixed-cluster' { + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'true' + setting 'xpack.watcher.enabled', 'false' + setting 'xpack.monitoring.enabled', 'false' + setting 'xpack.license.self_generated.type', 'trial' + setting 'cluster.remote.my_remote_cluster.seeds', { + testClusters.'remote-cluster'.getAllTransportPortURI().collect { "\"$it\"" }.toString() + } + setting 'cluster.remote.connections_per_cluster', "1" + setting 'cluster.remote.connect', "true" + + user username: "test_user", password: "x-pack-test-password" +} + +task integTest { + dependsOn 'mixed-cluster' +} + +test.enabled = false // no unit tests for multi-cluster-search, only the rest integration test +check.dependsOn(integTest) diff --git a/x-pack/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java b/x-pack/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java new file mode 100644 index 00000000000..740ebeff7b4 --- /dev/null +++ b/x-pack/qa/multi-cluster-tests-with-security/src/test/java/org/elasticsearch/multi_cluster/MultiClusterYamlTestSuiteIT.java @@ -0,0 +1,50 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.multi_cluster; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; +import com.carrotsearch.randomizedtesting.annotations.TimeoutSuite; + +import org.apache.lucene.util.TimeUnits; +import org.elasticsearch.common.settings.SecureString; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; + +import static org.elasticsearch.xpack.test.SecuritySettingsSourceField.basicAuthHeaderValue; + +@TimeoutSuite(millis = 5 * TimeUnits.MINUTE) // to account for slow as hell VMs +public class MultiClusterYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + private static final String USER = "test_user"; + private static final String PASS = "x-pack-test-password"; + + @Override + protected boolean preserveIndicesUponCompletion() { + return true; + } + + public MultiClusterYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws Exception { + return createParameters(); + } + + @Override + protected Settings restClientSettings() { + String token = basicAuthHeaderValue(USER, new SecureString(PASS.toCharArray())); + return Settings.builder() + .put(super.restClientSettings()) + .put(ThreadContext.PREFIX + ".Authorization", token) + .build(); + } +} diff --git a/x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml b/x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml new file mode 100644 index 00000000000..fb1cbda58fa --- /dev/null +++ b/x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/multi_cluster/80_transform.yml @@ -0,0 +1,261 @@ +--- +setup: + - skip: + features: headers + + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_user: + username: "joe" + body: > + { + "password": "transform", + "roles" : [ "transform_admin", "x_cluster_role" ] + } + - do: + security.put_role: + name: "x_cluster_role" + body: > + { + "cluster": [], + "indices": [ + { + "names": ["test_index", "my_remote_cluster:test_i*", "my_remote_cluster:aliased_test_index"], + "privileges": ["all", "view_index_metadata"] + }, + { + "names": ["simple-remote-transform", "simple-local-remote-transform"], + "privileges": ["all"] + } + ] + } +--- +teardown: + - do: + security.delete_user: + username: "joe" + ignore: 404 + +--- +"Search remote cluster": + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + search: + rest_total_hits_as_int: true + index: my_remote_cluster:test_index + body: + aggs: + user: + terms: + field: user + + - match: { _shards.total: 3 } + - match: { hits.total: 9 } + - length: { aggregations.user.buckets: 3 } + - match: { aggregations.user.buckets.0.key: "a" } + - match: { aggregations.user.buckets.0.doc_count: 5 } + +--- +"Batch transform from remote cluster": + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.put_transform: + transform_id: "simple-remote-transform" + body: > + { + "source": { "index": "my_remote_cluster:test_index" }, + "dest": { "index": "simple-remote-transform" }, + "pivot": { + "group_by": { "user": {"terms": {"field": "user"}}}, + "aggs": {"avg_stars": {"avg": {"field": "stars"}}} + } + } + - match: { acknowledged: true } + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.start_transform: + transform_id: "simple-remote-transform" + - match: { acknowledged: true } + + - do: + transform.get_transform_stats: + transform_id: "simple-remote-transform" + - match: { count: 1 } + - match: { transforms.0.id: "simple-remote-transform" } + - match: { transforms.0.state: "/started|indexing|stopping|stopped/" } + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.stop_transform: + transform_id: "simple-remote-transform" + wait_for_completion: true + wait_for_checkpoint: true + - match: { acknowledged: true } + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.get_transform_stats: + transform_id: "simple-remote-transform" + - match: { count: 1 } + - match: { transforms.0.id: "simple-remote-transform" } + - match: { transforms.0.state: "stopped" } + - match: { transforms.0.checkpointing.last.checkpoint: 1 } + + # workaround: refresh dest index, to be removed, see gh #51154 + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + indices.refresh: + index: simple-remote-transform + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + search: + rest_total_hits_as_int: true + index: simple-remote-transform + sort: user + + - match: { hits.total: 3 } + - match: { hits.hits.0._index: simple-remote-transform } + - match: { hits.hits.0._source.avg_stars: 3.6 } + - match: { hits.hits.0._source.user: a } + - match: { hits.hits.1._source.avg_stars: 2.0 } + - match: { hits.hits.1._source.user: b } + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.update_transform: + transform_id: "simple-remote-transform" + body: > + { + "source": { "index": ["my_remote_cluster:test_index", "my_remote_cluster:test_index_2"] } + } + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.get_transform_stats: + transform_id: "simple-remote-transform" + - match: { count: 1 } + - match: { transforms.0.id: "simple-remote-transform" } + - match: { transforms.0.state: "stopped" } + # we added test_index_2, which has 2 more docs: + - match: { transforms.0.checkpointing.operations_behind: 2 } + +--- +"Batch transform from local and remote cluster": + - do: + indices.create: + index: test_index + body: + settings: + index: + number_of_shards: 3 + number_of_replicas: 0 + aliases: + test_alias: {} + mappings: + properties: + time: + type: date + user: + type: keyword + stars: + type: integer + coolness: + type: integer + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index"}}' + - '{"user": "a", "stars": 3, "date" : "2018-11-29T12:12:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "c", "stars": 5, "date" : "2018-11-29T12:14:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "d", "stars": 5, "date" : "2018-11-29T12:16:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "e", "stars": 2, "date" : "2018-11-29T12:17:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "b", "stars": 3, "date" : "2018-11-29T12:22:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "c", "stars": 5, "date" : "2018-11-29T12:23:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "d", "stars": 1, "date" : "2018-11-29T12:32:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "e", "stars": 3, "date" : "2018-11-29T12:34:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "c", "stars": 4, "date" : "2018-11-29T12:35:12.123456789Z"}' + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.put_transform: + transform_id: "simple-local-remote-transform" + body: > + { + "source": { "index": ["test_index", "my_remote_cluster:test_index"] }, + "dest": { "index": "simple-local-remote-transform" }, + "pivot": { + "group_by": { "user": {"terms": {"field": "user"}}}, + "aggs": { + "avg_stars": {"avg": {"field": "stars"}}, + "count": {"value_count": {"field": "user"}} + } + } + } + - match: { acknowledged: true } + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.start_transform: + transform_id: "simple-local-remote-transform" + - match: { acknowledged: true } + + - do: + transform.get_transform_stats: + transform_id: "simple-local-remote-transform" + - match: { count: 1 } + - match: { transforms.0.id: "simple-local-remote-transform" } + - match: { transforms.0.state: "/started|indexing|stopping|stopped/" } + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.stop_transform: + transform_id: "simple-local-remote-transform" + wait_for_completion: true + wait_for_checkpoint: true + - match: { acknowledged: true } + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + transform.get_transform_stats: + transform_id: "simple-local-remote-transform" + - match: { count: 1 } + - match: { transforms.0.id: "simple-local-remote-transform" } + - match: { transforms.0.state: "stopped" } + - match: { transforms.0.checkpointing.last.checkpoint: 1 } + + # workaround: refresh dest index, to be removed, see gh #51154 + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + indices.refresh: + index: simple-local-remote-transform + + - do: + headers: { Authorization: "Basic am9lOnRyYW5zZm9ybQ==" } + search: + rest_total_hits_as_int: true + index: simple-local-remote-transform + sort: user + + - match: { hits.total: 5 } + - match: { hits.hits.0._index: simple-local-remote-transform } + - match: { hits.hits.0._source.avg_stars: 3.5 } + - match: { hits.hits.0._source.count: 6 } + - match: { hits.hits.0._source.user: a } + - match: { hits.hits.1._source.avg_stars: 2.25 } + - match: { hits.hits.1._source.user: b } + - match: { hits.hits.3._source.count: 2 } + - match: { hits.hits.3._source.user: d } diff --git a/x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml b/x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml new file mode 100644 index 00000000000..46d15c05c01 --- /dev/null +++ b/x-pack/qa/multi-cluster-tests-with-security/src/test/resources/rest-api-spec/test/remote_cluster/80_transform.yml @@ -0,0 +1,126 @@ +--- +setup: + - skip: + features: headers + + - do: + cluster.health: + wait_for_status: yellow + - do: + security.put_user: + username: "joe" + body: > + { + "password": "transform", + "roles" : [ "x_cluster_role" ] + } + - do: + security.put_role: + name: "x_cluster_role" + body: > + { + "cluster": [], + "indices": [ + { + "names": ["test_index*"], + "privileges": ["read", "view_index_metadata"] + } + ] + } +--- +teardown: + - do: + security.delete_user: + username: "joe" + ignore: 404 + +--- +"Index data on the remote cluster": + - do: + indices.create: + index: test_index + body: + settings: + index: + number_of_shards: 3 + number_of_replicas: 0 + aliases: + test_alias: {} + mappings: + properties: + time: + type: date + user: + type: keyword + stars: + type: integer + coolness: + type: integer + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index"}}' + - '{"user": "a", "stars": 1, "date" : "2018-10-29T12:12:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "a", "stars": 4, "date" : "2018-10-29T12:14:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "a", "stars": 5, "date" : "2018-10-29T12:16:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "b", "stars": 2, "date" : "2018-10-29T12:17:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "b", "stars": 3, "date" : "2018-10-29T12:22:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "a", "stars": 5, "date" : "2018-10-29T12:23:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "b", "stars": 1, "date" : "2018-10-29T12:32:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "a", "stars": 3, "date" : "2018-10-29T12:34:12.123456789Z"}' + - '{"index": {"_index": "test_index"}}' + - '{"user": "c", "stars": 4, "date" : "2018-10-29T12:35:12.123456789Z"}' + - do: + search: + rest_total_hits_as_int: true + index: test_index + body: + aggs: + user: + terms: + field: user + + - match: { _shards.total: 3 } + - match: { hits.total: 9 } + - length: { aggregations.user.buckets: 3 } + - match: { aggregations.user.buckets.0.key: "a" } + - match: { aggregations.user.buckets.0.doc_count: 5 } + + - do: + indices.create: + index: test_index_2 + body: + settings: + index: + number_of_shards: 3 + number_of_replicas: 0 + aliases: + test_alias: {} + mappings: + properties: + time: + type: date + user: + type: keyword + stars: + type: integer + coolness: + type: integer + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "test_index_2"}}' + - '{"user": "e", "stars": 3, "date" : "2018-10-29T12:12:12.123456789Z"}' + - '{"index": {"_index": "test_index_2"}}' + - '{"user": "d", "stars": 4, "date" : "2018-10-29T12:14:12.123456789Z"}'