From 833e0f8ecf43be368da2490d71aaa2c88785b5d7 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 6 Nov 2018 09:45:30 +0100 Subject: [PATCH] Prevent throttled indices to be searched through wildcards by default (#34354) Today if a wildcard, date-math expression or alias expands/resolves to an index that is search-throttled we still search it. This is likely not the desired behavior since it can unexpectedly slow down searches significantly. This change adds a new indices option that allows `search`, `count` and `msearch` to ignore throttled indices by default. Users can force expansion to throttled indices by using `ignore_throttled=true` on the rest request to expand also to throttled indices. Relates to #34352 --- .../org/elasticsearch/client/RankEvalIT.java | 2 +- .../client/RequestConvertersTests.java | 3 +- .../ExplainLifecycleRequestTests.java | 4 +- ...emoveIndexLifecyclePolicyRequestTests.java | 6 +- .../mustache/MultiSearchTemplateRequest.java | 2 +- .../index/rankeval/RankEvalRequestIT.java | 14 +-- .../index/rankeval/RankEvalRequestTests.java | 3 +- .../resources/rest-api-spec/api/count.json | 4 + .../resources/rest-api-spec/api/search.json | 4 + .../rest-api-spec/api/search_template.json | 4 + .../indices/alias/IndicesAliasesRequest.java | 2 +- .../indices/delete/DeleteIndexRequest.java | 2 +- .../action/search/MultiSearchRequest.java | 4 +- .../search/MultiSearchRequestBuilder.java | 7 +- .../action/search/SearchRequest.java | 2 +- .../action/support/IndicesOptions.java | 52 ++++++++++-- .../metadata/IndexNameExpressionResolver.java | 21 ++++- .../CanMatchPreFilterSearchPhaseTests.java | 7 +- .../search/MultiSearchRequestTests.java | 16 ++-- .../action/search/SearchAsyncActionTests.java | 7 +- .../search/TransportSearchActionTests.java | 8 +- .../action/support/IndicesOptionsTests.java | 62 +++++++++----- .../IndexNameExpressionResolverTests.java | 85 +++++++++++++++++-- .../WildcardExpressionResolverTests.java | 12 +-- .../search/SearchServiceTests.java | 15 ++++ .../ExplainLifecycleRequestTests.java | 4 +- ...emoveIndexLifecyclePolicyRequestTests.java | 4 +- .../action/RetryRequestTests.java | 4 +- 28 files changed, 268 insertions(+), 92 deletions(-) diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java index 15272ad80a6..9a0a861cc13 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RankEvalIT.java @@ -108,7 +108,7 @@ public class RankEvalIT extends ESRestHighLevelClientTestCase { // now try this when test2 is closed client().performRequest(new Request("POST", "index2/_close")); - rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + rankEvalRequest.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = execute(rankEvalRequest, highLevelClient()::rankEval, highLevelClient()::rankEvalAsync); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 1e0f0f70b2f..066fb5d8cc9 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -1055,7 +1055,8 @@ public class RequestConvertersTests extends ESTestCase { IndicesOptions msearchDefault = new MultiSearchRequest().indicesOptions(); searchRequest.indicesOptions(IndicesOptions.fromOptions(randomlyGenerated.ignoreUnavailable(), randomlyGenerated.allowNoIndices(), randomlyGenerated.expandWildcardsOpen(), randomlyGenerated.expandWildcardsClosed(), - msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases())); + msearchDefault.allowAliasesToMultipleIndices(), msearchDefault.forbidClosedIndices(), msearchDefault.ignoreAliases(), + msearchDefault.ignoreThrottled())); multiSearchRequest.add(searchRequest); } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java index 11063943339..933503e629b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/ExplainLifecycleRequestTests.java @@ -38,7 +38,7 @@ public class ExplainLifecycleRequestTests extends ESTestCase { } if (randomBoolean()) { IndicesOptions indicesOptions = IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), - randomBoolean(), randomBoolean(), randomBoolean()); + randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); request.indicesOptions(indicesOptions); } return request; @@ -54,7 +54,7 @@ public class ExplainLifecycleRequestTests extends ESTestCase { break; case 1: indicesOptions = randomValueOtherThan(indicesOptions, () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), - randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); break; default: throw new AssertionError("Illegal randomisation branch"); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java index 532688c4751..d5ccabc748d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/indexlifecycle/RemoveIndexLifecyclePolicyRequestTests.java @@ -46,7 +46,7 @@ public class RemoveIndexLifecyclePolicyRequestTests extends ESTestCase { if (randomBoolean()) { return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false)), IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), - randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); + randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())); } else { return new RemoveIndexLifecyclePolicyRequest(Arrays.asList(generateRandomStringArray(20, 20, false))); } @@ -57,14 +57,14 @@ public class RemoveIndexLifecyclePolicyRequestTests extends ESTestCase { req.indicesOptions().ignoreUnavailable(), req.indicesOptions().allowNoIndices(), req.indicesOptions().expandWildcardsOpen(), req.indicesOptions().expandWildcardsClosed(), req.indicesOptions().allowAliasesToMultipleIndices(), req.indicesOptions().forbidClosedIndices(), - req.indicesOptions().ignoreAliases())); + req.indicesOptions().ignoreAliases(), req.indicesOptions().ignoreThrottled())); } private RemoveIndexLifecyclePolicyRequest mutateInstance(RemoveIndexLifecyclePolicyRequest req) { if (randomBoolean()) { return new RemoveIndexLifecyclePolicyRequest(req.indices(), randomValueOtherThan(req.indicesOptions(), () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), - randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()))); + randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()))); } else { return new RemoveIndexLifecyclePolicyRequest( randomValueOtherThan(req.indices(), () -> Arrays.asList(generateRandomStringArray(20, 20, false))), diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java index eea9e31d4a7..a685c3ba5ba 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/MultiSearchTemplateRequest.java @@ -45,7 +45,7 @@ public class MultiSearchTemplateRequest extends ActionRequest implements Composi private int maxConcurrentSearchRequests = 0; private List requests = new ArrayList<>(); - private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); + private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); /** * Add a search template request to execute. Note, the order is important, the search response will be returned in the diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java index cdad280fd9a..cc5b554e39e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java @@ -286,7 +286,7 @@ public class RankEvalRequestIT extends ESIntegTestCase { // test that ignore_unavailable=true works but returns one result less assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged()); - request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, "true", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(6, details.getRetrieved()); @@ -294,37 +294,37 @@ public class RankEvalRequestIT extends ESIntegTestCase { // test that ignore_unavailable=false or default settings throw an IndexClosedException assertTrue(client().admin().indices().prepareClose("test2").get().isAcknowledged()); - request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, "false", null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class)); // test expand_wildcards request = new RankEvalRequest(task, new String[] { "tes*" }); - request.indicesOptions(IndicesOptions.fromParameters("none", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters("none", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(0, details.getRetrieved()); - request.indicesOptions(IndicesOptions.fromParameters("open", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters("open", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(6, details.getRetrieved()); assertEquals(5, details.getRelevantRetrieved()); - request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters("closed", null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexClosedException.class)); // test allow_no_indices request = new RankEvalRequest(task, new String[] { "bad*" }); - request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, null, "true", "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); details = (PrecisionAtK.Detail) response.getPartialResults().get("amsterdam_query").getMetricDetails(); assertEquals(0, details.getRetrieved()); - request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); + request.indicesOptions(IndicesOptions.fromParameters(null, null, "false", "false", SearchRequest.DEFAULT_INDICES_OPTIONS)); response = client().execute(RankEvalAction.INSTANCE, request).actionGet(); assertEquals(1, response.getFailures().size()); assertThat(response.getFailures().get("amsterdam_query"), instanceOf(IndexNotFoundException.class)); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 10e3611b30d..1a16c311fcf 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -59,7 +59,8 @@ public class RankEvalRequestTests extends AbstractWireSerializingTestCase private String[] indices; // Delete index should work by default on both open and closed indices. - private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true); + private IndicesOptions indicesOptions = IndicesOptions.fromOptions(false, true, true, true, false, false, true, false); public DeleteIndexRequest() { } diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java index 056c4c29c7a..0cff8aadb52 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequest.java @@ -58,7 +58,7 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice private int maxConcurrentSearchRequests = 0; private List requests = new ArrayList<>(); - private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosed(); + private IndicesOptions indicesOptions = IndicesOptions.strictExpandOpenAndForbidClosedIgnoreThrottled(); /** * Add a search request to execute. Note, the order is important, the search response will be returned in the @@ -287,7 +287,7 @@ public class MultiSearchRequest extends ActionRequest implements CompositeIndice } return output.toByteArray(); } - + public static void writeSearchRequestParams(SearchRequest request, XContentBuilder xContentBuilder) throws IOException { xContentBuilder.startObject(); if (request.indices() != null) { diff --git a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java index 64d512f4be0..ce43f47a497 100644 --- a/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/MultiSearchRequestBuilder.java @@ -28,6 +28,7 @@ import org.elasticsearch.client.ElasticsearchClient; */ public class MultiSearchRequestBuilder extends ActionRequestBuilder { + public MultiSearchRequestBuilder(ElasticsearchClient client, MultiSearchAction action) { super(client, action, new MultiSearchRequest()); } @@ -40,7 +41,8 @@ public class MultiSearchRequestBuilder extends ActionRequestBuilder NONE = EnumSet.noneOf(Option.class); } @@ -94,6 +95,9 @@ public class IndicesOptions implements ToXContentFragment { new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES), EnumSet.of(WildcardStates.OPEN, WildcardStates.CLOSED)); public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED = new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES), EnumSet.of(WildcardStates.OPEN)); + public static final IndicesOptions STRICT_EXPAND_OPEN_FORBID_CLOSED_IGNORE_THROTTLED = + new IndicesOptions(EnumSet.of(Option.ALLOW_NO_INDICES, Option.FORBID_CLOSED_INDICES, Option.IGNORE_THROTTLED), + EnumSet.of(WildcardStates.OPEN)); public static final IndicesOptions STRICT_SINGLE_INDEX_NO_EXPAND_FORBID_CLOSED = new IndicesOptions(EnumSet.of(Option.FORBID_ALIASES_TO_MULTIPLE_INDICES, Option.FORBID_CLOSED_INDICES), EnumSet.noneOf(WildcardStates.class)); @@ -242,7 +246,21 @@ public class IndicesOptions implements ToXContentFragment { return options.contains(Option.IGNORE_ALIASES); } + /** + * + * @return whether indices that are marked as throttled should be ignored when resolving a wildcard or alias + */ + public boolean ignoreThrottled() { + return options.contains(Option.IGNORE_THROTTLED); + } + public void writeIndicesOptions(StreamOutput out) throws IOException { + EnumSet