From 84ee21ed26feed2e6e4337c550b533b29048a453 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 12 Jul 2017 22:19:34 +0200 Subject: [PATCH] Followup for elastic/elasticsearch#25658 (elastic/x-pack-elasticsearch#1984) This is the xpack side fo elastic/elasticsearch#25658 which is mainly refactorings of a ctor and added tests. Original commit: elastic/x-pack-elasticsearch@d8e2a2a0577c7eb7171a4fc1ba3eecc3a9a44f33 --- .../security/InternalClientIntegTests.java | 2 +- .../CompareConditionSearchTests.java | 2 +- .../condition/ScriptConditionSearchTests.java | 2 +- .../condition/ScriptConditionTests.java | 12 +-- .../execution/TriggeredWatchStoreTests.java | 4 +- .../test/integration/SearchInputTests.java | 9 +- .../org/elasticsearch/transport/handlers | 2 + .../test/multi_cluster/60_skip_shards.yml | 96 +++++++++++++++++++ .../test/remote_cluster/10_basic.yml | 26 ++++- 9 files changed, 140 insertions(+), 15 deletions(-) create mode 100644 qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml diff --git a/plugin/src/test/java/org/elasticsearch/xpack/security/InternalClientIntegTests.java b/plugin/src/test/java/org/elasticsearch/xpack/security/InternalClientIntegTests.java index 7e87b39279b..660c7a29134 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/security/InternalClientIntegTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/security/InternalClientIntegTests.java @@ -75,7 +75,7 @@ public class InternalClientIntegTests extends ESSingleNodeTestCase { String scrollId = randomAlphaOfLength(5); SearchHit[] hits = new SearchHit[] {new SearchHit(1)}; InternalSearchResponse internalResponse = new InternalSearchResponse(new SearchHits(hits, 1, 1), null, null, null, false, false, 1); - SearchResponse response = new SearchResponse(internalResponse, scrollId, 1, 1, 0, ShardSearchFailure.EMPTY_ARRAY); + SearchResponse response = new SearchResponse(internalResponse, scrollId, 1, 1, 0, 0, ShardSearchFailure.EMPTY_ARRAY); Answer returnResponse = invocation -> { @SuppressWarnings("unchecked") diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java index d457c948cc7..0b97f11d9f2 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/CompareConditionSearchTests.java @@ -84,7 +84,7 @@ public class CompareConditionSearchTests extends AbstractWatcherIntegrationTestC InternalSearchResponse internalSearchResponse = new InternalSearchResponse( new SearchHits(new SearchHit[]{hit}, 1L, 1f), null, null, null, false, false, 1); - SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response)); assertThat(condition.execute(ctx).met(), is(true)); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionSearchTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionSearchTests.java index cc5c8ba7522..d5ded151220 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionSearchTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionSearchTests.java @@ -107,7 +107,7 @@ public class ScriptConditionSearchTests extends AbstractWatcherIntegrationTestCa InternalSearchResponse internalSearchResponse = new InternalSearchResponse(new SearchHits( new SearchHit[]{hit}, 1L, 1f), null, null, null, false, false, 1); - SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(internalSearchResponse, "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_watch_name", new Payload.XContent(response)); assertThat(condition.execute(ctx).met(), is(true)); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java index 7483d5a90c3..4f3fc2a31a8 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/condition/ScriptConditionTests.java @@ -94,7 +94,7 @@ public class ScriptConditionTests extends ESTestCase { public void testExecute() throws Exception { ScriptCondition condition = new ScriptCondition(mockScript("ctx.payload.hits.total > 1"), scriptService); - SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); assertFalse(condition.execute(ctx).met()); } @@ -102,7 +102,7 @@ public class ScriptConditionTests extends ESTestCase { public void testExecuteMergedParams() throws Exception { Script script = new Script(ScriptType.INLINE, "mockscript", "ctx.payload.hits.total > threshold", singletonMap("threshold", 1)); ScriptCondition executable = new ScriptCondition(script, scriptService); - SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); assertFalse(executable.execute(ctx).met()); } @@ -115,7 +115,7 @@ public class ScriptConditionTests extends ESTestCase { parser.nextToken(); ScriptCondition executable = ScriptCondition.parse(scriptService, "_watch", parser); - SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); assertFalse(executable.execute(ctx).met()); @@ -179,7 +179,7 @@ public class ScriptConditionTests extends ESTestCase { public void testScriptConditionThrowException() throws Exception { ScriptCondition condition = new ScriptCondition( mockScript("null.foo"), scriptService); - SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); ScriptException exception = expectThrows(ScriptException.class, () -> condition.execute(ctx)); assertThat(exception.getMessage(), containsString("Error evaluating null.foo")); @@ -187,7 +187,7 @@ public class ScriptConditionTests extends ESTestCase { public void testScriptConditionReturnObjectThrowsException() throws Exception { ScriptCondition condition = new ScriptCondition(mockScript("return new Object()"), scriptService); - SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_name", new Payload.XContent(response)); Exception exception = expectThrows(IllegalStateException.class, () -> condition.execute(ctx)); assertThat(exception.getMessage(), @@ -197,7 +197,7 @@ public class ScriptConditionTests extends ESTestCase { public void testScriptConditionAccessCtx() throws Exception { ScriptCondition condition = new ScriptCondition(mockScript("ctx.trigger.scheduled_time.getMillis() < new Date().time"), scriptService); - SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 500L, new ShardSearchFailure[0]); + SearchResponse response = new SearchResponse(InternalSearchResponse.empty(), "", 3, 3, 0, 500L, new ShardSearchFailure[0]); WatchExecutionContext ctx = mockExecutionContext("_name", new DateTime(DateTimeZone.UTC), new Payload.XContent(response)); Thread.sleep(10); assertThat(condition.execute(ctx).met(), is(true)); diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java index e4a03bdcd77..7774b2bf96a 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/execution/TriggeredWatchStoreTests.java @@ -196,8 +196,8 @@ public class TriggeredWatchStoreTests extends ESTestCase { hit.sourceRef(source); hits = new SearchHits(new SearchHit[]{hit}, 1, 1.0f); SearchResponse searchResponse2 = new SearchResponse( - new InternalSearchResponse(hits, null, null, null, false, null, 1), "_scrollId1", 1, 1, 1, null); - SearchResponse searchResponse3 = new SearchResponse(InternalSearchResponse.empty(), "_scrollId2", 1, 1, 1, null); + new InternalSearchResponse(hits, null, null, null, false, null, 1), "_scrollId1", 1, 1, 0, 1, null); + SearchResponse searchResponse3 = new SearchResponse(InternalSearchResponse.empty(), "_scrollId2", 1, 1, 0, 1, null); doAnswer(invocation -> { SearchScrollRequest request = (SearchScrollRequest) invocation.getArguments()[0]; diff --git a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java index 43d051b9be9..d69bc42974e 100644 --- a/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java +++ b/plugin/src/test/java/org/elasticsearch/xpack/watcher/test/integration/SearchInputTests.java @@ -94,7 +94,8 @@ public class SearchInputTests extends ESTestCase { public void testExecute() throws Exception { ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(SearchRequest.class); PlainActionFuture searchFuture = PlainActionFuture.newFuture(); - SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 1234, ShardSearchFailure.EMPTY_ARRAY); + SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 0, 1234, + ShardSearchFailure.EMPTY_ARRAY); searchFuture.onResponse(searchResponse); when(client.search(requestCaptor.capture())).thenReturn(searchFuture); @@ -133,7 +134,8 @@ public class SearchInputTests extends ESTestCase { public void testDifferentSearchType() throws Exception { ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(SearchRequest.class); PlainActionFuture searchFuture = PlainActionFuture.newFuture(); - SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 1234, ShardSearchFailure.EMPTY_ARRAY); + SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 0, 1234, + ShardSearchFailure.EMPTY_ARRAY); searchFuture.onResponse(searchResponse); when(client.search(requestCaptor.capture())).thenReturn(searchFuture); @@ -174,7 +176,8 @@ public class SearchInputTests extends ESTestCase { public void testThatEmptyRequestBodyWorks() throws Exception { ArgumentCaptor requestCaptor = ArgumentCaptor.forClass(SearchRequest.class); PlainActionFuture searchFuture = PlainActionFuture.newFuture(); - SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 1234, ShardSearchFailure.EMPTY_ARRAY); + SearchResponse searchResponse = new SearchResponse(InternalSearchResponse.empty(), "", 1, 1, 0, 1234, + ShardSearchFailure.EMPTY_ARRAY); searchFuture.onResponse(searchResponse); when(client.search(requestCaptor.capture())).thenReturn(searchFuture); diff --git a/plugin/src/test/resources/org/elasticsearch/transport/handlers b/plugin/src/test/resources/org/elasticsearch/transport/handlers index 916a48af102..2cd94ca2580 100644 --- a/plugin/src/test/resources/org/elasticsearch/transport/handlers +++ b/plugin/src/test/resources/org/elasticsearch/transport/handlers @@ -64,6 +64,7 @@ indices:data/read/search[phase/query+fetch/scroll] indices:data/read/search[phase/query/id] indices:data/read/search[phase/query/scroll] indices:data/read/search[phase/query] +indices:data/read/search[can_match] internal:transport/proxy/indices:data/read/search[clear_scroll_contexts] internal:transport/proxy/indices:data/read/search[free_context/scroll] internal:transport/proxy/indices:data/read/search[free_context] @@ -74,6 +75,7 @@ internal:transport/proxy/indices:data/read/search[phase/query+fetch/scroll] internal:transport/proxy/indices:data/read/search[phase/query/id] internal:transport/proxy/indices:data/read/search[phase/query/scroll] internal:transport/proxy/indices:data/read/search[phase/query] +internal:transport/proxy/indices:data/read/search[can_match] indices:data/read/tv[s] indices:data/write/bulk[s] indices:data/write/bulk[s][p] diff --git a/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml b/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml new file mode 100644 index 00000000000..ad27f58567a --- /dev/null +++ b/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/multi_cluster/60_skip_shards.yml @@ -0,0 +1,96 @@ +--- +setup: + - skip: + features: headers + + - do: + cluster.health: + wait_for_status: yellow + - do: + xpack.security.put_user: + username: "joe" + body: > + { + "password": "s3krit", + "roles" : [ "x_cluster_role" ] + } + - do: + xpack.security.put_role: + name: "x_cluster_role" + body: > + { + "cluster": ["all"], + "indices": [ + { + "names": ["skip_shards_index", "my_remote_cluster:single_doc_index"], + "privileges": ["read"] + } + ] + } +--- +teardown: + - do: + xpack.security.delete_user: + username: "joe" + ignore: 404 + - do: + xpack.security.delete_role: + name: "x_cluster_role" + ignore: 404 +--- +"Test that remote indices are subject to shard skipping": + + - do: + indices.create: + index: skip_shards_index + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + test_type: + properties: + created_at: + type: date + format: "yyyy-MM-dd" + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "skip_shards_index", "_type": "test_type"}}' + - '{"f1": "local_cluster", "sort_field": 0, "created_at" : "2017-01-01"}' + + # check that we skip the remote shard + - do: + headers: { Authorization: "Basic am9lOnMza3JpdA==" } + search: + index: "skip_shards_index,my_remote_cluster:single_doc_index" + pre_filter_shard_size: 1 + body: { "size" : 10, "query" : { "range" : { "created_at" : { "gte" : "2016-02-01", "lt": "2018-02-01"} } } } + + - match: { hits.total: 1 } + - match: { hits.hits.0._index: "skip_shards_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + - match: { hits.total: 1 } + + # check that we skip the local shard + - do: + headers: { Authorization: "Basic am9lOnMza3JpdA==" } + search: + index: "skip_shards_index,my_remote_cluster:single_doc_index" + pre_filter_shard_size: 1 + body: { "size" : 10, "query" : { "range" : { "created_at" : { "gte" : "2015-02-01", "lt": "2016-02-01"} } } } + + - match: { hits.total: 1 } + - match: { hits.hits.0._index: "my_remote_cluster:single_doc_index"} + - match: { _shards.total: 2 } + - match: { _shards.successful: 2 } + - match: { _shards.skipped : 1} + - match: { _shards.failed: 0 } + - match: { hits.total: 1 } + diff --git a/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml b/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml index 9aca1eba67e..6fa2b1e31a1 100644 --- a/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml +++ b/qa/multi-cluster-search-security/src/test/resources/rest-api-spec/test/remote_cluster/10_basic.yml @@ -22,13 +22,37 @@ setup: "cluster": ["monitor"], "indices": [ { - "names": ["secure_alias", "test_index", "aliased_test_index", "field_caps_index_1", "field_caps_index_3"], + "names": ["single_doc_index", "secure_alias", "test_index", "aliased_test_index", "field_caps_index_1", + "field_caps_index_3"], "privileges": ["read", "read_cross_cluster"] } ] } --- "Index data and search on the remote cluster": + + - do: + indices.create: + index: single_doc_index + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + test_type: + properties: + created_at: + type: date + format: "yyyy-MM-dd" + + - do: + bulk: + refresh: true + body: + - '{"index": {"_index": "single_doc_index", "_type": "test_type"}}' + - '{"f1": "remote_cluster", "sort_field": 1, "created_at" : "2016-01-01"}' + - do: indices.create: index: field_caps_index_1