From 8a7f3f75f3eefde29fed0d0d414a798c50a25ee6 Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Thu, 29 Nov 2018 18:36:16 +0100 Subject: [PATCH] Add support for rest_total_hits_as_int (#36051) The support for rest_total_hits_as_int has already been merged to 6x in #35848 so this change adds this new option to master. The plan was to add this new option as part of #35848 but we've decided to wait a few days before merging this breaking change so this commit just handles the new option as a noop exactly like 6x for now. This will allow users to migrate to this parameter before #35848 is merged. Relates #33028 --- .../RestMultiSearchTemplateAction.java | 12 +++- .../mustache/RestSearchTemplateAction.java | 11 +++- .../test/lang_mustache/30_search_template.yml | 27 ++++++++ .../50_multi_search_template.yml | 44 +++++++++++++ .../resources/rest-api-spec/api/msearch.json | 5 ++ .../rest-api-spec/api/msearch_template.json | 5 ++ .../resources/rest-api-spec/api/scroll.json | 5 ++ .../resources/rest-api-spec/api/search.json | 5 ++ .../rest-api-spec/api/search_template.json | 5 ++ .../rest-api-spec/test/msearch/10_basic.yml | 25 ++++++++ .../rest-api-spec/test/scroll/10_basic.yml | 64 +++++++++++++++++++ .../test/search/20_default_values.yml | 10 +++ .../action/search/RestMultiSearchAction.java | 11 +++- .../rest/action/search/RestSearchAction.java | 9 ++- .../action/search/RestSearchScrollAction.java | 9 +++ .../rollup/rest/RestRollupSearchAction.java | 9 +++ .../test/rollup/rollup_search.yml | 27 ++++++++ 17 files changed, 279 insertions(+), 4 deletions(-) diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index cbed41d43b0..7091c520ddb 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -31,7 +31,9 @@ import org.elasticsearch.rest.action.search.RestMultiSearchAction; import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -40,7 +42,15 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestMultiSearchTemplateAction extends BaseRestHandler { private static final DeprecationLogger deprecationLogger = new DeprecationLogger( LogManager.getLogger(RestMultiSearchAction.class)); - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>( + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + ); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } private final boolean allowExplicitIndex; diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index e9bbc7aeeec..b352f4aadb0 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -30,7 +30,9 @@ import org.elasticsearch.rest.action.RestStatusToXContentListener; import org.elasticsearch.rest.action.search.RestSearchAction; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -38,7 +40,14 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestSearchTemplateAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>( + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + ); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } public RestSearchTemplateAction(Settings settings, RestController controller) { super(settings); diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml index 89205e973fa..f14004d3ebd 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/30_search_template.yml @@ -124,3 +124,30 @@ - match: { hits.total: 1 } - length: { hits.hits: 1 } - length: { profile: 1 } + +--- +"Test with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + + - do: + index: + index: test + type: type + id: 1 + body: {} + + - do: + put_script: + id: "template_1" + body: { "script": { "lang": "mustache", "source": { "query": { "match_all": {} } } } } + + - match: { acknowledged: true } + + - do: + search_template: + rest_total_hits_as_int: true + body: { "id": "template_1", "params": {} } + + - match: { hits.total: 0 } diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml index 0d0b16cdbfc..ae026d9d7a0 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/50_multi_search_template.yml @@ -156,3 +156,47 @@ setup: - match: { responses.0.hits.total: 2 } - match: { responses.1.hits.total: 1 } - match: { responses.2.hits.total: 1 } + + + - do: + put_script: + id: stored_template_1 + body: { "script": { "lang" : "mustache", "source": { "query": {"match": {"{{field}}": "{{value}}" }}}}} + - match: { acknowledged: true } + +--- +"Test with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + + - do: + put_script: + id: stored_template_1 + body: { "script": { "lang": "mustache", "source": { "query": {"match": {"{{field}}": "{{value}}" }}}}} + - match: { acknowledged: true } + + - do: + msearch_template: + rest_total_hits_as_int: true + body: + - index: index_* + - id: stored_template_1 + params: + field: "foo" + value: "foo" + - index: _all + - id: stored_template_1 + params: + field: "foo" + value: "bar" + - index: index_2 + - id: stored_template_1 + params: + field: "foo" + value: "foo" + + - match: { responses.0.hits.total: 2 } + - match: { responses.1.hits.total: 1 } + - match: { responses.2.hits.total: 1 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json index 13281a2a232..4acc769a97e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch.json @@ -38,6 +38,11 @@ "type" : "number", "description" : "The number of concurrent shard requests each sub search executes concurrently. This value should be used to limit the impact of the search on the cluster in order to limit the number of concurrent shard requests", "default" : "The default grows with the number of nodes in the cluster but is at most 256." + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json index e7420932932..dd02ed80786 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/msearch_template.json @@ -28,6 +28,11 @@ "max_concurrent_searches" : { "type" : "number", "description" : "Controls the maximum number of concurrent searches the multi search api will execute" + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json index 699ddcc9e00..f03653fa2f4 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/scroll.json @@ -19,6 +19,11 @@ "scroll_id": { "type" : "string", "description" : "The scroll ID for scrolled search" + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json index 5a576e8ce3c..da9bcb82f3b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search.json @@ -182,6 +182,11 @@ "type" : "number", "description" : "A threshold that enforces a pre-filter roundtrip to prefilter search shards based on query rewriting if the number of shards the search request expands to exceeds the threshold. This filter roundtrip can limit the number of shards significantly if for instance a shard can not match any documents based on it's rewrite method ie. if date filters are mandatory to match but the shard bounds and the query are disjoint.", "default" : 128 + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json index df1fc1b0799..c5c9770222b 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/search_template.json @@ -62,6 +62,11 @@ "typed_keys": { "type" : "boolean", "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response" + }, + "rest_total_hits_as_int" : { + "type" : "boolean", + "description" : "This parameter is ignored in this version. It is used in the next major version to control whether the rest response should render the total.hits as an object or a number", + "default" : false } } }, diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml index c16844478f1..557492623a8 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml @@ -93,3 +93,28 @@ setup: - match: { responses.3.error.root_cause.0.reason: "/no.such.index/" } - match: { responses.3.error.root_cause.0.index: index_3 } - match: { responses.4.hits.total: 4 } + +--- +"Search with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + + - do: + msearch: + rest_total_hits_as_int: true + body: + - index: index_* + - query: + match: {foo: foo} + - index: index_2 + - query: + match_all: {} + - index: index_1 + - query: + match: {foo: foo} + + - match: { responses.0.hits.total: 2 } + - match: { responses.1.hits.total: 1 } + - match: { responses.2.hits.total: 1 } + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml index 6ab18146bba..d4a4e4ee462 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/scroll/10_basic.yml @@ -281,3 +281,67 @@ - length: {hits.hits: 1 } - match: { hits.max_score: null } + +--- +"Scroll with rest_total_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + - do: + indices.create: + index: test_scroll + - do: + index: + index: test_scroll + type: test + id: 42 + body: { foo: 1 } + + - do: + index: + index: test_scroll + type: test + id: 43 + body: { foo: 2 } + + - do: + indices.refresh: {} + + - do: + search: + index: test_scroll + size: 1 + scroll: 1m + sort: foo + rest_total_hits_as_int: true + body: + query: + match_all: {} + + - set: {_scroll_id: scroll_id} + - match: {hits.total: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._id: "42" } + + - do: + scroll: + rest_total_hits_as_int: true + body: { "scroll_id": "$scroll_id", "scroll": "1m"} + + - match: {hits.total: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._id: "43" } + + - do: + scroll: + rest_total_hits_as_int: true + scroll_id: $scroll_id + scroll: 1m + + - match: {hits.total: 2 } + - length: {hits.hits: 0 } + + - do: + clear_scroll: + scroll_id: $scroll_id + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml index 359d5d80c68..0f42b1bd6b5 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/20_default_values.yml @@ -72,3 +72,13 @@ setup: match: foo: bar +--- +"Search with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + - do: + search: + rest_total_hits_as_int: true + + - match: {hits.total: 2} diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java index 475eec8ef8e..3080b50b99b 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java @@ -40,7 +40,9 @@ import org.elasticsearch.rest.action.RestToXContentListener; import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; +import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -53,7 +55,14 @@ public class RestMultiSearchAction extends BaseRestHandler { public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + " Specifying types in multi search requests is deprecated."; - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM); + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>( + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + ); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } private final boolean allowExplicitIndex; diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 7047a6b2aa0..65d640cf310 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -45,6 +45,7 @@ import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode; import java.io.IOException; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.Set; import java.util.function.IntConsumer; @@ -55,7 +56,13 @@ import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion; public class RestSearchAction extends BaseRestHandler { public static final String TYPED_KEYS_PARAM = "typed_keys"; - private static final Set RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM); + public static final String TOTAL_HIT_AS_INT_PARAM = "rest_total_hits_as_int"; + private static final Set RESPONSE_PARAMS; + + static { + final Set responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, TOTAL_HIT_AS_INT_PARAM)); + RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); + } private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(RestSearchAction.class)); public static final String TYPES_DEPRECATION_MESSAGE = "[types removal]" + diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java index bc3b0ccb56a..67f70b129c1 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java @@ -29,12 +29,16 @@ import org.elasticsearch.rest.action.RestStatusToXContentListener; import org.elasticsearch.search.Scroll; import java.io.IOException; +import java.util.Collections; +import java.util.Set; import static org.elasticsearch.common.unit.TimeValue.parseTimeValue; import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestSearchScrollAction extends BaseRestHandler { + Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM); + public RestSearchScrollAction(Settings settings, RestController controller) { super(settings); @@ -70,4 +74,9 @@ public class RestSearchScrollAction extends BaseRestHandler { }}); return channel -> client.searchScroll(searchScrollRequest, new RestStatusToXContentListener<>(channel)); } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } } diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java index 30638623ec9..e2bbfe9d8b6 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java @@ -16,9 +16,13 @@ import org.elasticsearch.rest.action.search.RestSearchAction; import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction; import java.io.IOException; +import java.util.Collections; +import java.util.Set; public class RestRollupSearchAction extends BaseRestHandler { + private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM); + public RestRollupSearchAction(Settings settings, RestController controller) { super(settings); controller.registerHandler(RestRequest.Method.GET, "_rollup_search", this); @@ -39,4 +43,9 @@ public class RestRollupSearchAction extends BaseRestHandler { public String getName() { return "rollup_search_action"; } + + @Override + protected Set responseParams() { + return RESPONSE_PARAMS; + } } diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml index 851d1cfa172..3399376b6ab 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml @@ -154,6 +154,33 @@ setup: - match: { aggregations.histo.buckets.3.key_as_string: "2017-01-01T08:00:00.000Z" } - match: { aggregations.histo.buckets.3.doc_count: 20 } +--- +"Basic Search with rest_total_hits_as_int": + - skip: + version: " - 6.5.99" + reason: rest_total_hits_as_int was introduced in 6.6.0 + - do: + xpack.rollup.rollup_search: + index: "foo_rollup" + body: + size: 0 + aggs: + histo: + date_histogram: + field: "timestamp" + interval: "1h" + time_zone: "UTC" + + - length: { aggregations.histo.buckets: 4 } + - match: { aggregations.histo.buckets.0.key_as_string: "2017-01-01T05:00:00.000Z" } + - match: { aggregations.histo.buckets.0.doc_count: 1 } + - match: { aggregations.histo.buckets.1.key_as_string: "2017-01-01T06:00:00.000Z" } + - match: { aggregations.histo.buckets.1.doc_count: 2 } + - match: { aggregations.histo.buckets.2.key_as_string: "2017-01-01T07:00:00.000Z" } + - match: { aggregations.histo.buckets.2.doc_count: 10 } + - match: { aggregations.histo.buckets.3.key_as_string: "2017-01-01T08:00:00.000Z" } + - match: { aggregations.histo.buckets.3.doc_count: 20 } + --- "Formatted Date Histo":