From 270291878022b2e09bdb0279c7b4443cf7668386 Mon Sep 17 00:00:00 2001 From: Lee Hinman Date: Fri, 8 Sep 2017 13:37:55 -0600 Subject: [PATCH] Limit the number of expanded fields it query_string and simple_query_string (#26541) * Limit the number of expanded fields it query_string and simple_query_string This limits the number of automatically expanded fields for the "all fields" mode (`"default_field": "*"`) for the `query_string` and `simple_query_string` queries to 1024 fields. Resolves #25105 * Add blurb about limit to the docs --- .../index/search/QueryParserHelper.java | 8 +++++ .../search/query/QueryStringIT.java | 33 +++++++++++++++++ .../search/query/SimpleQueryStringIT.java | 36 +++++++++++++++++++ .../query-dsl/query-string-query.asciidoc | 12 +++---- .../simple-query-string-query.asciidoc | 7 ++-- 5 files changed, 87 insertions(+), 9 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java b/core/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java index bc1a62f6939..e6846543b8c 100644 --- a/core/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java +++ b/core/src/main/java/org/elasticsearch/index/search/QueryParserHelper.java @@ -124,6 +124,7 @@ public final class QueryParserHelper { !multiField, !allField, fieldSuffix); resolvedFields.putAll(fieldMap); } + checkForTooManyFields(resolvedFields); return resolvedFields; } @@ -184,6 +185,13 @@ public final class QueryParserHelper { } fields.put(fieldName, weight); } + checkForTooManyFields(fields); return fields; } + + private static void checkForTooManyFields(Map fields) { + if (fields.size() > 1024) { + throw new IllegalArgumentException("field expansion matches too many fields, limit: 1024, got: " + fields.size()); + } + } } diff --git a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java index 651d8a8fc83..a44d6ff6d5e 100644 --- a/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/QueryStringIT.java @@ -28,6 +28,7 @@ import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.index.query.QueryStringQueryBuilder; @@ -46,6 +47,7 @@ import java.util.HashSet; import java.util.List; import java.util.Set; +import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.queryStringQuery; import static org.elasticsearch.test.StreamsUtils.copyToStringFromClasspath; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; @@ -351,6 +353,37 @@ public class QueryStringIT extends ESIntegTestCase { assertSearchHits(searchResponse, "1", "2", "3"); } + public void testLimitOnExpandedFields() throws Exception { + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + builder.startObject("type1"); + builder.startObject("properties"); + for (int i = 0; i < 1025; i++) { + builder.startObject("field" + i).field("type", "text").endObject(); + } + builder.endObject(); // properties + builder.endObject(); // type1 + builder.endObject(); + + assertAcked(prepareCreate("toomanyfields") + .setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200)) + .addMapping("type1", builder)); + + client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get(); + refresh(); + + Exception e = expectThrows(Exception.class, () -> { + QueryStringQueryBuilder qb = queryStringQuery("bar"); + if (randomBoolean()) { + qb.useAllFields(true); + } + logger.info("--> using {}", qb); + client().prepareSearch("toomanyfields").setQuery(qb).get(); + }); + assertThat(ExceptionsHelper.detailedMessage(e), + containsString("field expansion matches too many fields, limit: 1024, got: 1025")); + } + private void assertHits(SearchHits hits, String... ids) { assertThat(hits.getTotalHits(), equalTo((long) ids.length)); Set hitIds = new HashSet<>(); diff --git a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java index b06bdf6cdc9..398b30abbe1 100644 --- a/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java +++ b/core/src/test/java/org/elasticsearch/search/query/SimpleQueryStringIT.java @@ -24,11 +24,15 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.admin.indices.create.CreateIndexRequestBuilder; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.mapper.MapperService; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.Operator; import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.index.query.SimpleQueryStringBuilder; import org.elasticsearch.index.query.SimpleQueryStringFlag; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.SearchHit; @@ -540,6 +544,38 @@ public class SimpleQueryStringIT extends ESIntegTestCase { containsString("NumberFormatException[For input string: \"foo123\"]")); } + + public void testLimitOnExpandedFields() throws Exception { + XContentBuilder builder = jsonBuilder(); + builder.startObject(); + builder.startObject("type1"); + builder.startObject("properties"); + for (int i = 0; i < 1025; i++) { + builder.startObject("field" + i).field("type", "text").endObject(); + } + builder.endObject(); // properties + builder.endObject(); // type1 + builder.endObject(); + + assertAcked(prepareCreate("toomanyfields") + .setSettings(Settings.builder().put(MapperService.INDEX_MAPPING_TOTAL_FIELDS_LIMIT_SETTING.getKey(), 1200)) + .addMapping("type1", builder)); + + client().prepareIndex("toomanyfields", "type1", "1").setSource("field171", "foo bar baz").get(); + refresh(); + + Exception e = expectThrows(Exception.class, () -> { + SimpleQueryStringBuilder qb = simpleQueryStringQuery("bar"); + if (randomBoolean()) { + qb.useAllFields(true); + } + logger.info("--> using {}", qb); + client().prepareSearch("toomanyfields").setQuery(qb).get(); + }); + assertThat(ExceptionsHelper.detailedMessage(e), + containsString("field expansion matches too many fields, limit: 1024, got: 1025")); + } + private void assertHits(SearchHits hits, String... ids) { assertThat(hits.getTotalHits(), equalTo((long) ids.length)); Set hitIds = new HashSet<>(); diff --git a/docs/reference/query-dsl/query-string-query.asciidoc b/docs/reference/query-dsl/query-string-query.asciidoc index 2b91c9cd6e1..f42a3b09f9b 100644 --- a/docs/reference/query-dsl/query-string-query.asciidoc +++ b/docs/reference/query-dsl/query-string-query.asciidoc @@ -48,12 +48,12 @@ The `query_string` top level parameters include: |Parameter |Description |`query` |The actual query to be parsed. See <>. -|`default_field` |The default field for query terms if no prefix field -is specified. Defaults to the `index.query.default_field` index -settings, which in turn defaults to `*`. -`*` extracts all fields in the mapping that are eligible to term queries -and filters the metadata fields. All extracted fields are then combined -to build a query when no prefix field is provided. +|`default_field` |The default field for query terms if no prefix field is +specified. Defaults to the `index.query.default_field` index settings, which in +turn defaults to `*`. `*` extracts all fields in the mapping that are eligible +to term queries and filters the metadata fields. All extracted fields are then +combined to build a query when no prefix field is provided. There is a limit of +no more than 1024 fields being queried at once. |`default_operator` |The default operator used if no explicit operator is specified. For example, with a default operator of `OR`, the query diff --git a/docs/reference/query-dsl/simple-query-string-query.asciidoc b/docs/reference/query-dsl/simple-query-string-query.asciidoc index 68a47193a14..1f887cf6311 100644 --- a/docs/reference/query-dsl/simple-query-string-query.asciidoc +++ b/docs/reference/query-dsl/simple-query-string-query.asciidoc @@ -29,9 +29,10 @@ The `simple_query_string` top level parameters include: |`query` |The actual query to be parsed. See below for syntax. |`fields` |The fields to perform the parsed query against. Defaults to the -`index.query.default_field` index settings, which in turn defaults to `*`. -`*` extracts all fields in the mapping that are eligible to term queries -and filters the metadata fields. +`index.query.default_field` index settings, which in turn defaults to `*`. `*` +extracts all fields in the mapping that are eligible to term queries and filters +the metadata fields. There is a limit of no more than 1024 fields being queried +at once. |`default_operator` |The default operator used if no explicit operator is specified. For example, with a default operator of `OR`, the query