From e38cf1d0dca1776d9659b4c0f3c010ba444642ca Mon Sep 17 00:00:00 2001 From: Jim Ferenczi Date: Fri, 4 Jan 2019 20:36:49 +0100 Subject: [PATCH] Add the ability to set the number of hits to track accurately (#36357) In Lucene 8 searches can skip non-competitive hits if the total hit count is not requested. It is also possible to track the number of hits up to a certain threshold. This is a trade off to speed up searches while still being able to know a lower bound of the total hit count. This change adds the ability to set this threshold directly in the track_total_hits search option. A boolean value (true, false) indicates whether the total hit count should be tracked in the response. When set as an integer this option allows to compute a lower bound of the total hits while preserving the ability to skip non-competitive hits when enough matches have been collected. Relates #33028 --- docs/reference/search/request-body.asciidoc | 2 + .../search/request/track-total-hits.asciidoc | 176 ++++++++++++++++++ docs/reference/search/uri-request.asciidoc | 8 +- .../RestMultiSearchTemplateAction.java | 3 +- .../mustache/RestSearchTemplateAction.java | 3 +- .../elasticsearch/upgrades/IndexingIT.java | 4 +- .../rest-api-spec/test/msearch/10_basic.yml | 40 +++- .../test/search/220_total_hits_object.yml | 100 +++++++++- .../search/AbstractSearchAsyncAction.java | 7 +- .../action/search/SearchPhaseController.java | 54 ++++-- .../action/search/SearchRequestBuilder.java | 8 + .../action/search/RestMultiSearchAction.java | 3 +- .../rest/action/search/RestSearchAction.java | 33 +++- .../action/search/RestSearchScrollAction.java | 2 +- .../search/DefaultSearchContext.java | 10 +- .../org/elasticsearch/search/SearchHits.java | 23 ++- .../elasticsearch/search/SearchService.java | 2 +- .../search/builder/SearchSourceBuilder.java | 53 ++++-- .../internal/FilteredSearchContext.java | 8 +- .../internal/InternalSearchResponse.java | 7 +- .../search/internal/SearchContext.java | 11 +- .../search/query/QueryPhase.java | 4 +- .../search/query/TopDocsCollectorContext.java | 89 +++++---- .../search/SearchPhaseControllerTests.java | 11 +- .../action/search/SearchRequestTests.java | 1 + .../search/query/QueryPhaseTests.java | 3 +- .../search/RandomSearchRequestGenerator.java | 9 +- .../elasticsearch/test/TestSearchContext.java | 10 +- .../xpack/ccr/ESCCRRestTestCase.java | 4 +- .../rollup/rest/RestRollupSearchAction.java | 3 +- .../xpack/test/rest/XPackRestIT.java | 6 +- .../xpack/restart/FullClusterRestartIT.java | 8 +- .../elasticsearch/upgrades/IndexingIT.java | 4 +- .../upgrades/RollupIDUpgradeIT.java | 4 +- .../SmokeTestWatcherWithSecurityIT.java | 4 +- .../SmokeTestWatcherTestSuiteIT.java | 4 +- 36 files changed, 573 insertions(+), 148 deletions(-) create mode 100644 docs/reference/search/request/track-total-hits.asciidoc diff --git a/docs/reference/search/request-body.asciidoc b/docs/reference/search/request-body.asciidoc index 7145b40c43e..9970c4cc622 100644 --- a/docs/reference/search/request-body.asciidoc +++ b/docs/reference/search/request-body.asciidoc @@ -189,6 +189,8 @@ include::request/from-size.asciidoc[] include::request/sort.asciidoc[] +include::request/track-total-hits.asciidoc[] + include::request/source-filtering.asciidoc[] include::request/stored-fields.asciidoc[] diff --git a/docs/reference/search/request/track-total-hits.asciidoc b/docs/reference/search/request/track-total-hits.asciidoc new file mode 100644 index 00000000000..bdad4dbde91 --- /dev/null +++ b/docs/reference/search/request/track-total-hits.asciidoc @@ -0,0 +1,176 @@ +[[search-request-track-total-hits]] +=== Track total hits + +Generally the total hit count can't be computed accurately without visiting all +matches, which is costly for queries that match lots of documents. The +`track_total_hits` parameter allows you to control how the total number of hits +should be tracked. When set to `true` the search response will always track the +number of hits that match the query accurately (e.g. `total.relation` will always +be equal to `"eq"` when `track_total_hits is set to true). + +[source,js] +-------------------------------------------------- +GET twitter/_search +{ + "track_total_hits": true, + "query": { + "match" : { + "message" : "Elasticsearch" + } + } +} +-------------------------------------------------- +// TEST[setup:twitter] +// CONSOLE + +\... returns: + +[source,js] +-------------------------------------------------- +{ + "_shards": ... + "timed_out": false, + "took": 100, + "hits": { + "max_score": 1.0, + "total" : { + "value": 2048, <1> + "relation": "eq" <2> + }, + "hits": ... + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards",/] +// TESTRESPONSE[s/"took": 100/"took": $body.took/] +// TESTRESPONSE[s/"max_score": 1\.0/"max_score": $body.hits.max_score/] +// TESTRESPONSE[s/"value": 2048/"value": $body.hits.total.value/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": "$body.hits.hits"/] + +<1> The total number of hits that match the query. +<2> The count is accurate (e.g. `"eq"` means equals). + +If you don't need to track the total number of hits you can improve query times +by setting this option to `false`. In such case the search can efficiently skip +non-competitive hits because it doesn't need to count all matches: + +[source,js] +-------------------------------------------------- +GET twitter/_search +{ + "track_total_hits": false, + "query": { + "match" : { + "message" : "Elasticsearch" + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +\... returns: + +[source,js] +-------------------------------------------------- +{ + "_shards": ... + "timed_out": false, + "took": 10, + "hits" : { <1> + "max_score": 1.0, + "hits": ... + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards",/] +// TESTRESPONSE[s/"took": 10/"took": $body.took/] +// TESTRESPONSE[s/"max_score": 1\.0/"max_score": $body.hits.max_score/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": "$body.hits.hits"/] + +<1> The total number of hits is unknown. + +Given that it is often enough to have a lower bound of the number of hits, +such as "there are at least 1000 hits", it is also possible to set +`track_total_hits` as an integer that represents the number of hits to count +accurately. The search can efficiently skip non-competitive document as soon +as collecting at least $`track_total_hits` documents. This is a good trade +off to speed up searches if you don't need the accurate number of hits after +a certain threshold. + + +For instance the following query will track the total hit count that match +the query accurately up to 100 documents: + +[source,js] +-------------------------------------------------- +GET twitter/_search +{ + "track_total_hits": 100, + "query": { + "match" : { + "message" : "Elasticsearch" + } + } +} +-------------------------------------------------- +// CONSOLE +// TEST[continued] + +The `hits.total.relation` in the response will indicate if the +value returned in `hits.total.value` is accurate (`eq`) or a lower +bound of the total (`gte`). + +For instance the following response: + +[source,js] +-------------------------------------------------- +{ + "_shards": ... + "timed_out": false, + "took": 30, + "hits" : { + "max_score": 1.0, + "total" : { + "value": 42, <1> + "relation": "eq" <2> + }, + "hits": ... + } +} +-------------------------------------------------- +// TESTRESPONSE[s/"_shards": \.\.\./"_shards": "$body._shards",/] +// TESTRESPONSE[s/"took": 30/"took": $body.took/] +// TESTRESPONSE[s/"max_score": 1\.0/"max_score": $body.hits.max_score/] +// TESTRESPONSE[s/"value": 42/"value": $body.hits.total.value/] +// TESTRESPONSE[s/"hits": \.\.\./"hits": "$body.hits.hits"/] + +<1> 42 documents match the query +<2> and the count is accurate (`"eq"`) + +\... indicates that the number of hits returned in the `total` +is accurate. + +If the total number of his that match the query is greater than the +value set in `track_total_hits`, the total hits in the response +will indicate that the returned value is a lower bound: + +[source,js] +-------------------------------------------------- +{ + "_shards": ... + "hits" : { + "max_score": 1.0, + "total" : { + "value": 100, <1> + "relation": "gte" <2> + }, + "hits": ... + } +} +-------------------------------------------------- +// TESTRESPONSE +// TEST[skip:response is already tested in the previous snippet] + +<1> There are at least 100 documents that match the query +<2> This is a lower bound (`gte`). \ No newline at end of file diff --git a/docs/reference/search/uri-request.asciidoc b/docs/reference/search/uri-request.asciidoc index 320e65bf3ee..87e1da907fb 100644 --- a/docs/reference/search/uri-request.asciidoc +++ b/docs/reference/search/uri-request.asciidoc @@ -101,10 +101,12 @@ is important). |`track_scores` |When sorting, set to `true` in order to still track scores and return them as part of each hit. -|`track_total_hits` |Set to `false` in order to disable the tracking +|`track_total_hits` |Defaults to true. Set to `false` in order to disable the tracking of the total number of hits that match the query. -(see <> for more details). -Defaults to true. +It also accepts an integer which in this case represents the number of +hits to count accurately. +(See the <> documentation +for more details). |`timeout` |A search timeout, bounding the search request to be executed within the specified time value and bail with the hits accumulated up to diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java index 30543ea236a..77733223574 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java @@ -49,7 +49,7 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler { static { final Set responseParams = new HashSet<>( - Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM) ); RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); } @@ -103,6 +103,7 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler { } else { throw new IllegalArgumentException("Malformed search template"); } + RestSearchAction.checkRestTotalHits(restRequest, searchRequest); }); return multiRequest; } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java index 196147bb730..70a12f0c8bf 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/RestSearchTemplateAction.java @@ -43,7 +43,7 @@ public class RestSearchTemplateAction extends BaseRestHandler { private static final Set RESPONSE_PARAMS; static { - final Set responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM)); + final Set responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM)); RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); } @@ -77,6 +77,7 @@ public class RestSearchTemplateAction extends BaseRestHandler { searchTemplateRequest = SearchTemplateRequest.fromXContent(parser); } searchTemplateRequest.setRequest(searchRequest); + RestSearchAction.checkRestTotalHits(request, searchRequest); return channel -> client.execute(SearchTemplateAction.INSTANCE, searchTemplateRequest, new RestStatusToXContentListener<>(channel)); } diff --git a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index 1d8872da4f5..e4736a79d05 100644 --- a/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -30,7 +30,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.hamcrest.Matchers.equalTo; /** @@ -158,7 +158,7 @@ public class IndexingIT extends AbstractRollingTestCase { private void assertCount(String index, int count) throws IOException { Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); - searchTestIndexRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); searchTestIndexRequest.addParameter("filter_path", "hits.total"); Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest); assertEquals("{\"hits\":{\"total\":" + count + "}}", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml index 0d178a389be..f9fe244529f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/msearch/10_basic.yml @@ -115,11 +115,45 @@ setup: - query: match: {foo: foo} - - match: { responses.0.hits.total.value: 2 } + - match: { responses.0.hits.total.value: 2 } - match: { responses.0.hits.total.relation: eq } - - match: { responses.1.hits.total.value: 1 } + - match: { responses.1.hits.total.value: 1 } - match: { responses.1.hits.total.relation: eq } - - match: { responses.2.hits.total.value: 1 } + - match: { responses.2.hits.total.value: 1 } - match: { responses.2.hits.total.relation: eq } + - do: + msearch: + body: + - index: index_* + - { query: { match: {foo: foo}}, track_total_hits: 1 } + - index: index_2 + - query: + match_all: {} + - index: index_1 + - query: + match: {foo: foo} + + - match: { responses.0.hits.total.value: 1 } + - match: { responses.0.hits.total.relation: gte } + - match: { responses.1.hits.total.value: 1 } + - match: { responses.1.hits.total.relation: eq } + - match: { responses.2.hits.total.value: 1 } + - match: { responses.2.hits.total.relation: eq } + + - do: + catch: /\[rest_total_hits_as_int\] cannot be used if the tracking of total hits is not accurate, got 10/ + msearch: + rest_total_hits_as_int: true + body: + - index: index_* + - { query: { match_all: {}}, track_total_hits: 10} + - index: index_2 + - query: + match_all: {} + - index: index_1 + - query: + match: {foo: foo} + + diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml index f1660980869..a9c37c00b92 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/220_total_hits_object.yml @@ -2,9 +2,11 @@ setup: - do: indices.create: index: test_2 + - do: indices.create: index: test_1 + - do: index: index: test_1 @@ -14,10 +16,45 @@ setup: - do: index: - index: test_2 - type: test - id: 42 - body: { foo: bar } + index: test_1 + type: test + id: 3 + body: { foo: baz } + + - do: + index: + index: test_1 + type: test + id: 2 + body: { foo: bar } + + - do: + index: + index: test_1 + type: test + id: 4 + body: { foo: bar } + + - do: + index: + index: test_2 + type: test + id: 42 + body: { foo: bar } + + - do: + index: + index: test_2 + type: test + id: 24 + body: { foo: baz } + + - do: + index: + index: test_2 + type: test + id: 36 + body: { foo: bar } - do: indices.refresh: @@ -28,6 +65,7 @@ setup: - skip: version: " - 6.99.99" reason: hits.total is rendered as an object in 7.0.0 + - do: search: index: _all @@ -36,7 +74,7 @@ setup: match: foo: bar - - match: {hits.total.value: 2} + - match: {hits.total.value: 5} - match: {hits.total.relation: eq} - do: @@ -47,7 +85,7 @@ setup: match: foo: bar - - match: {hits.total.value: 1} + - match: {hits.total.value: 3} - match: {hits.total.relation: eq} - do: @@ -61,6 +99,54 @@ setup: - is_false: hits.total + - do: + search: + track_total_hits: 4 + body: + query: + match: + foo: bar + + - match: {hits.total.value: 4} + - match: {hits.total.relation: gte} + + + - do: + search: + size: 3 + track_total_hits: 4 + body: + query: + match: + foo: bar + + - match: {hits.total.value: 4} + - match: {hits.total.relation: gte} + + - do: + catch: /\[rest_total_hits_as_int\] cannot be used if the tracking of total hits is not accurate, got 100/ + search: + rest_total_hits_as_int: true + index: test_2 + track_total_hits: 100 + body: + query: + match: + foo: bar + + - do: + catch: /\[track_total_hits\] parameter must be positive or equals to -1, got -2/ + search: + rest_total_hits_as_int: true + index: test_2 + track_total_hits: -2 + body: + query: + match: + foo: bar + +--- +"track_total_hits with rest_total_hits_as_int": - do: search: track_total_hits: false @@ -82,6 +168,6 @@ setup: match: foo: bar - - match: {hits.total: 1} + - match: {hits.total: 2} diff --git a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java index 7091fb6bef2..f031dfa5810 100644 --- a/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java +++ b/server/src/main/java/org/elasticsearch/action/search/AbstractSearchAsyncAction.java @@ -34,6 +34,7 @@ import org.elasticsearch.search.SearchPhaseResult; import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.AliasFilter; import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.internal.ShardSearchTransportRequest; import org.elasticsearch.transport.Transport; @@ -113,7 +114,11 @@ abstract class AbstractSearchAsyncAction exten if (getNumShards() == 0) { //no search shards to search on, bail with empty response //(it happens with search across _all with no indices around and consistent with broadcast operations) - listener.onResponse(new SearchResponse(InternalSearchResponse.empty(), null, 0, 0, 0, buildTookInMillis(), + + boolean withTotalHits = request.source() != null ? + // total hits is null in the response if the tracking of total hits is disabled + request.source().trackTotalHitsUpTo() != SearchContext.TRACK_TOTAL_HITS_DISABLED : true; + listener.onResponse(new SearchResponse(InternalSearchResponse.empty(withTotalHits), null, 0, 0, 0, buildTookInMillis(), ShardSearchFailure.EMPTY_ARRAY, clusters)); return; } diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java index 66b0317146f..418d95b2077 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseController.java @@ -48,6 +48,7 @@ import org.elasticsearch.search.dfs.AggregatedDfs; import org.elasticsearch.search.dfs.DfsSearchResult; import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.profile.ProfileShardResult; import org.elasticsearch.search.profile.SearchProfileShardResults; import org.elasticsearch.search.query.QuerySearchResult; @@ -408,17 +409,17 @@ public final class SearchPhaseController { * @param queryResults a list of non-null query shard results */ ReducedQueryPhase reducedScrollQueryPhase(Collection queryResults) { - return reducedQueryPhase(queryResults, true, true, true); + return reducedQueryPhase(queryResults, true, SearchContext.TRACK_TOTAL_HITS_ACCURATE, true); } /** * Reduces the given query results and consumes all aggregations and profile results. * @param queryResults a list of non-null query shard results */ - ReducedQueryPhase reducedQueryPhase(Collection queryResults, - boolean isScrollRequest, boolean trackTotalHits, boolean performFinalReduce) { - return reducedQueryPhase(queryResults, null, new ArrayList<>(), new TopDocsStats(trackTotalHits), 0, isScrollRequest, - performFinalReduce); + public ReducedQueryPhase reducedQueryPhase(Collection queryResults, + boolean isScrollRequest, int trackTotalHitsUpTo, boolean performFinalReduce) { + return reducedQueryPhase(queryResults, null, new ArrayList<>(), new TopDocsStats(trackTotalHitsUpTo), + 0, isScrollRequest, performFinalReduce); } /** @@ -618,7 +619,7 @@ public final class SearchPhaseController { private int index; private final SearchPhaseController controller; private int numReducePhases = 0; - private final TopDocsStats topDocsStats = new TopDocsStats(); + private final TopDocsStats topDocsStats; private final boolean performFinalReduce; /** @@ -629,7 +630,7 @@ public final class SearchPhaseController { * the buffer is used to incrementally reduce aggregation results before all shards responded. */ private QueryPhaseResultConsumer(SearchPhaseController controller, int expectedResultSize, int bufferSize, - boolean hasTopDocs, boolean hasAggs, boolean performFinalReduce) { + boolean hasTopDocs, boolean hasAggs, int trackTotalHitsUpTo, boolean performFinalReduce) { super(expectedResultSize); if (expectedResultSize != 1 && bufferSize < 2) { throw new IllegalArgumentException("buffer size must be >= 2 if there is more than one expected result"); @@ -647,6 +648,7 @@ public final class SearchPhaseController { this.hasTopDocs = hasTopDocs; this.hasAggs = hasAggs; this.bufferSize = bufferSize; + this.topDocsStats = new TopDocsStats(trackTotalHitsUpTo); this.performFinalReduce = performFinalReduce; } @@ -718,47 +720,65 @@ public final class SearchPhaseController { boolean isScrollRequest = request.scroll() != null; final boolean hasAggs = source != null && source.aggregations() != null; final boolean hasTopDocs = source == null || source.size() != 0; - final boolean trackTotalHits = source == null || source.trackTotalHits(); + final int trackTotalHitsUpTo = source == null ? SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO : source.trackTotalHitsUpTo(); final boolean finalReduce = request.getLocalClusterAlias() == null; if (isScrollRequest == false && (hasAggs || hasTopDocs)) { // no incremental reduce if scroll is used - we only hit a single shard or sometimes more... if (request.getBatchedReduceSize() < numShards) { // only use this if there are aggs and if there are more shards than we should reduce at once - return new QueryPhaseResultConsumer(this, numShards, request.getBatchedReduceSize(), hasTopDocs, hasAggs, finalReduce); + return new QueryPhaseResultConsumer(this, numShards, request.getBatchedReduceSize(), hasTopDocs, hasAggs, + trackTotalHitsUpTo, finalReduce); } } return new InitialSearchPhase.ArraySearchPhaseResults(numShards) { @Override ReducedQueryPhase reduce() { - return reducedQueryPhase(results.asList(), isScrollRequest, trackTotalHits, finalReduce); + return reducedQueryPhase(results.asList(), isScrollRequest, trackTotalHitsUpTo, finalReduce); } }; } static final class TopDocsStats { - final boolean trackTotalHits; + final int trackTotalHitsUpTo; private long totalHits; private TotalHits.Relation totalHitsRelation; long fetchHits; float maxScore = Float.NEGATIVE_INFINITY; TopDocsStats() { - this(true); + this(SearchContext.TRACK_TOTAL_HITS_ACCURATE); } - TopDocsStats(boolean trackTotalHits) { - this.trackTotalHits = trackTotalHits; + TopDocsStats(int trackTotalHitsUpTo) { + this.trackTotalHitsUpTo = trackTotalHitsUpTo; this.totalHits = 0; - this.totalHitsRelation = trackTotalHits ? Relation.EQUAL_TO : Relation.GREATER_THAN_OR_EQUAL_TO; + this.totalHitsRelation = Relation.EQUAL_TO; } TotalHits getTotalHits() { - return trackTotalHits ? new TotalHits(totalHits, totalHitsRelation) : null; + if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_DISABLED) { + return null; + } else if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_ACCURATE) { + assert totalHitsRelation == Relation.EQUAL_TO; + return new TotalHits(totalHits, totalHitsRelation); + } else { + if (totalHits < trackTotalHitsUpTo) { + return new TotalHits(totalHits, totalHitsRelation); + } else { + /** + * The user requested to count the total hits up to trackTotalHitsUpTo + * so we return this lower bound when the total hits is greater than this value. + * This can happen when multiple shards are merged since the limit to track total hits + * is applied per shard. + */ + return new TotalHits(trackTotalHitsUpTo, Relation.GREATER_THAN_OR_EQUAL_TO); + } + } } void add(TopDocsAndMaxScore topDocs) { - if (trackTotalHits) { + if (trackTotalHitsUpTo != SearchContext.TRACK_TOTAL_HITS_DISABLED) { totalHits += topDocs.topDocs.totalHits.value; if (topDocs.topDocs.totalHits.relation == Relation.GREATER_THAN_OR_EQUAL_TO) { totalHitsRelation = TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO; diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java index 732debf2a13..e3d63cac050 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchRequestBuilder.java @@ -376,6 +376,14 @@ public class SearchRequestBuilder extends ActionRequestBuilder responseParams = new HashSet<>( - Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HIT_AS_INT_PARAM) + Arrays.asList(RestSearchAction.TYPED_KEYS_PARAM, RestSearchAction.TOTAL_HITS_AS_INT_PARAM) ); RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); } @@ -118,6 +118,7 @@ public class RestMultiSearchAction extends BaseRestHandler { deprecationLogger.deprecatedAndMaybeLog("msearch_with_types", TYPES_DEPRECATION_MESSAGE); } searchRequest.source(SearchSourceBuilder.fromXContent(parser, false)); + RestSearchAction.checkRestTotalHits(restRequest, searchRequest); multiRequest.add(searchRequest); }); List requests = multiRequest.requests(); diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java index 8e6e247123d..3e3a1e02a17 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java @@ -23,6 +23,7 @@ import org.apache.logging.log4j.LogManager; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.common.settings.Settings; @@ -59,12 +60,12 @@ public class RestSearchAction extends BaseRestHandler { * Indicates whether hits.total should be rendered as an integer or an object * in the rest search response. */ - public static final String TOTAL_HIT_AS_INT_PARAM = "rest_total_hits_as_int"; + public static final String TOTAL_HITS_AS_INT_PARAM = "rest_total_hits_as_int"; public static final String TYPED_KEYS_PARAM = "typed_keys"; private static final Set RESPONSE_PARAMS; static { - final Set responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, TOTAL_HIT_AS_INT_PARAM)); + final Set responseParams = new HashSet<>(Arrays.asList(TYPED_KEYS_PARAM, TOTAL_HITS_AS_INT_PARAM)); RESPONSE_PARAMS = Collections.unmodifiableSet(responseParams); } @@ -172,6 +173,7 @@ public class RestSearchAction extends BaseRestHandler { searchRequest.routing(request.param("routing")); searchRequest.preference(request.param("preference")); searchRequest.indicesOptions(IndicesOptions.fromRequest(request, searchRequest.indicesOptions())); + checkRestTotalHits(request, searchRequest); } /** @@ -236,7 +238,15 @@ public class RestSearchAction extends BaseRestHandler { } if (request.hasParam("track_total_hits")) { - searchSourceBuilder.trackTotalHits(request.paramAsBoolean("track_total_hits", true)); + if (Booleans.isBoolean(request.param("track_total_hits"))) { + searchSourceBuilder.trackTotalHits( + request.paramAsBoolean("track_total_hits", true) + ); + } else { + searchSourceBuilder.trackTotalHitsUpTo( + request.paramAsInt("track_total_hits", SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO) + ); + } } String sSorts = request.param("sort"); @@ -275,6 +285,23 @@ public class RestSearchAction extends BaseRestHandler { } } + /** + * Throws an {@link IllegalArgumentException} if {@link #TOTAL_HITS_AS_INT_PARAM} + * is used in conjunction with a lower bound value for the track_total_hits option. + */ + public static void checkRestTotalHits(RestRequest restRequest, SearchRequest searchRequest) { + int trackTotalHitsUpTo = searchRequest.source() == null ? + SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO : searchRequest.source().trackTotalHitsUpTo(); + if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_ACCURATE || + trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_DISABLED) { + return ; + } + if (restRequest.paramAsBoolean(TOTAL_HITS_AS_INT_PARAM, false)) { + throw new IllegalArgumentException("[" + TOTAL_HITS_AS_INT_PARAM + "] cannot be used " + + "if the tracking of total hits is not accurate, got " + trackTotalHitsUpTo); + } + } + @Override protected Set responseParams() { return RESPONSE_PARAMS; diff --git a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java index 6d2f0971ad7..50806a096f1 100644 --- a/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java +++ b/server/src/main/java/org/elasticsearch/rest/action/search/RestSearchScrollAction.java @@ -37,7 +37,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; public class RestSearchScrollAction extends BaseRestHandler { - private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM); + private static final Set RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HITS_AS_INT_PARAM); public RestSearchScrollAction(Settings settings, RestController controller) { super(settings); diff --git a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java index 05e90214e15..590c58b1f66 100644 --- a/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/DefaultSearchContext.java @@ -116,7 +116,7 @@ final class DefaultSearchContext extends SearchContext { private SortAndFormats sort; private Float minimumScore; private boolean trackScores = false; // when sorting, track scores as well... - private boolean trackTotalHits = true; + private int trackTotalHitsUpTo = SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO; private FieldDoc searchAfter; private CollapseContext collapse; private boolean lowLevelCancellation; @@ -558,14 +558,14 @@ final class DefaultSearchContext extends SearchContext { } @Override - public SearchContext trackTotalHits(boolean trackTotalHits) { - this.trackTotalHits = trackTotalHits; + public SearchContext trackTotalHitsUpTo(int trackTotalHitsUpTo) { + this.trackTotalHitsUpTo = trackTotalHitsUpTo; return this; } @Override - public boolean trackTotalHits() { - return trackTotalHits; + public int trackTotalHitsUpTo() { + return trackTotalHitsUpTo; } @Override diff --git a/server/src/main/java/org/elasticsearch/search/SearchHits.java b/server/src/main/java/org/elasticsearch/search/SearchHits.java index 6f295eff426..f04183ffde7 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchHits.java +++ b/server/src/main/java/org/elasticsearch/search/SearchHits.java @@ -44,10 +44,13 @@ import java.util.Objects; import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; public final class SearchHits implements Streamable, ToXContentFragment, Iterable { - public static SearchHits empty() { + return empty(true); + } + + public static SearchHits empty(boolean withTotalHits) { // We shouldn't use static final instance, since that could directly be returned by native transport clients - return new SearchHits(EMPTY, new TotalHits(0, Relation.EQUAL_TO), 0); + return new SearchHits(EMPTY, withTotalHits ? new TotalHits(0, Relation.EQUAL_TO) : null, 0); } public static final SearchHit[] EMPTY = new SearchHit[0]; @@ -151,7 +154,7 @@ public final class SearchHits implements Streamable, ToXContentFragment, Iterabl @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.HITS); - boolean totalHitAsInt = params.paramAsBoolean(RestSearchAction.TOTAL_HIT_AS_INT_PARAM, false); + boolean totalHitAsInt = params.paramAsBoolean(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, false); if (totalHitAsInt) { long total = totalHits == null ? -1 : totalHits.in.value; builder.field(Fields.TOTAL, total); @@ -329,12 +332,17 @@ public final class SearchHits implements Streamable, ToXContentFragment, Iterabl private static class Total implements Writeable, ToXContentFragment { final TotalHits in; + Total(TotalHits in) { + this.in = Objects.requireNonNull(in); + } + Total(StreamInput in) throws IOException { this.in = Lucene.readTotalHits(in); } - Total(TotalHits in) { - this.in = Objects.requireNonNull(in); + @Override + public void writeTo(StreamOutput out) throws IOException { + Lucene.writeTotalHits(out, in); } @Override @@ -351,11 +359,6 @@ public final class SearchHits implements Streamable, ToXContentFragment, Iterabl return Objects.hash(in.value, in.relation); } - @Override - public void writeTo(StreamOutput out) throws IOException { - Lucene.writeTotalHits(out, in); - } - @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.field("value", in.value); diff --git a/server/src/main/java/org/elasticsearch/search/SearchService.java b/server/src/main/java/org/elasticsearch/search/SearchService.java index 24cc4b45b74..f6e91c03af6 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchService.java +++ b/server/src/main/java/org/elasticsearch/search/SearchService.java @@ -814,7 +814,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv if (source.trackTotalHits() == false && context.scrollContext() != null) { throw new SearchContextException(context, "disabling [track_total_hits] is not allowed in a scroll context"); } - context.trackTotalHits(source.trackTotalHits()); + context.trackTotalHitsUpTo(source.trackTotalHitsUpTo()); if (source.minScore() != null) { context.minimumScore(source.minScore()); } diff --git a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java index a199ce3a377..73f6aa2fbff 100644 --- a/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/builder/SearchSourceBuilder.java @@ -22,6 +22,7 @@ package org.elasticsearch.search.builder; import org.apache.logging.log4j.LogManager; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.Version; +import org.elasticsearch.common.Booleans; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; @@ -68,6 +69,9 @@ import java.util.Objects; import java.util.stream.Collectors; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; +import static org.elasticsearch.search.internal.SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO; +import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_ACCURATE; +import static org.elasticsearch.search.internal.SearchContext.TRACK_TOTAL_HITS_DISABLED; /** * A search source builder allowing to easily build search source. Simple @@ -110,7 +114,6 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R public static final ParseField SEARCH_AFTER = new ParseField("search_after"); public static final ParseField COLLAPSE = new ParseField("collapse"); public static final ParseField SLICE = new ParseField("slice"); - public static final ParseField ALL_FIELDS_FIELDS = new ParseField("all_fields"); public static SearchSourceBuilder fromXContent(XContentParser parser) throws IOException { return fromXContent(parser, true); @@ -152,7 +155,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R private boolean trackScores = false; - private boolean trackTotalHits = true; + private int trackTotalHitsUpTo = DEFAULT_TRACK_TOTAL_HITS_UP_TO; private SearchAfterBuilder searchAfterBuilder; @@ -249,10 +252,10 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R searchAfterBuilder = in.readOptionalWriteable(SearchAfterBuilder::new); sliceBuilder = in.readOptionalWriteable(SliceBuilder::new); collapse = in.readOptionalWriteable(CollapseBuilder::new); - if (in.getVersion().onOrAfter(Version.V_6_0_0_beta1)) { - trackTotalHits = in.readBoolean(); + if (in.getVersion().onOrAfter(Version.V_7_0_0)) { + trackTotalHitsUpTo = in.readInt(); } else { - trackTotalHits = true; + trackTotalHitsUpTo = in.readBoolean() ? TRACK_TOTAL_HITS_ACCURATE : TRACK_TOTAL_HITS_DISABLED; } } @@ -312,8 +315,10 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R out.writeOptionalWriteable(searchAfterBuilder); out.writeOptionalWriteable(sliceBuilder); out.writeOptionalWriteable(collapse); - if (out.getVersion().onOrAfter(Version.V_6_0_0_beta1)) { - out.writeBoolean(trackTotalHits); + if (out.getVersion().onOrAfter(Version.V_7_0_0)) { + out.writeInt(trackTotalHitsUpTo); + } else { + out.writeBoolean(trackTotalHitsUpTo > SearchContext.TRACK_TOTAL_HITS_DISABLED); } } @@ -536,11 +541,24 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R * Indicates if the total hit count for the query should be tracked. */ public boolean trackTotalHits() { - return trackTotalHits; + return trackTotalHitsUpTo == TRACK_TOTAL_HITS_ACCURATE; } public SearchSourceBuilder trackTotalHits(boolean trackTotalHits) { - this.trackTotalHits = trackTotalHits; + this.trackTotalHitsUpTo = trackTotalHits ? TRACK_TOTAL_HITS_ACCURATE : TRACK_TOTAL_HITS_DISABLED; + return this; + } + + public int trackTotalHitsUpTo() { + return trackTotalHitsUpTo; + } + + public SearchSourceBuilder trackTotalHitsUpTo(int trackTotalHitsUpTo) { + if (trackTotalHitsUpTo < TRACK_TOTAL_HITS_DISABLED) { + throw new IllegalArgumentException("[track_total_hits] parameter must be positive or equals to -1, " + + "got " + trackTotalHitsUpTo); + } + this.trackTotalHitsUpTo = trackTotalHitsUpTo; return this; } @@ -979,7 +997,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R rewrittenBuilder.terminateAfter = terminateAfter; rewrittenBuilder.timeout = timeout; rewrittenBuilder.trackScores = trackScores; - rewrittenBuilder.trackTotalHits = trackTotalHits; + rewrittenBuilder.trackTotalHitsUpTo = trackTotalHitsUpTo; rewrittenBuilder.version = version; rewrittenBuilder.collapse = collapse; return rewrittenBuilder; @@ -1025,7 +1043,12 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R } else if (TRACK_SCORES_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { trackScores = parser.booleanValue(); } else if (TRACK_TOTAL_HITS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { - trackTotalHits = parser.booleanValue(); + if (token == XContentParser.Token.VALUE_BOOLEAN || + (token == XContentParser.Token.VALUE_STRING && Booleans.isBoolean(parser.text()))) { + trackTotalHitsUpTo = parser.booleanValue() ? TRACK_TOTAL_HITS_ACCURATE : TRACK_TOTAL_HITS_DISABLED; + } else { + trackTotalHitsUpTo = parser.intValue(); + } } else if (_SOURCE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { fetchSourceContext = FetchSourceContext.fromXContent(parser); } else if (STORED_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) { @@ -1231,8 +1254,8 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R builder.field(TRACK_SCORES_FIELD.getPreferredName(), true); } - if (trackTotalHits == false) { - builder.field(TRACK_TOTAL_HITS_FIELD.getPreferredName(), false); + if (trackTotalHitsUpTo != SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO) { + builder.field(TRACK_TOTAL_HITS_FIELD.getPreferredName(), trackTotalHitsUpTo); } if (searchAfterBuilder != null) { @@ -1500,7 +1523,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R return Objects.hash(aggregations, explain, fetchSourceContext, docValueFields, storedFieldsContext, from, highlightBuilder, indexBoosts, minScore, postQueryBuilder, queryBuilder, rescoreBuilders, scriptFields, size, sorts, searchAfterBuilder, sliceBuilder, stats, suggestBuilder, terminateAfter, timeout, trackScores, version, - profile, extBuilders, collapse, trackTotalHits); + profile, extBuilders, collapse, trackTotalHitsUpTo); } @Override @@ -1538,7 +1561,7 @@ public final class SearchSourceBuilder implements Writeable, ToXContentObject, R && Objects.equals(profile, other.profile) && Objects.equals(extBuilders, other.extBuilders) && Objects.equals(collapse, other.collapse) - && Objects.equals(trackTotalHits, other.trackTotalHits); + && Objects.equals(trackTotalHitsUpTo, other.trackTotalHitsUpTo); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java index 4f95fcc0195..3a7fb9f823f 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/FilteredSearchContext.java @@ -322,13 +322,13 @@ public abstract class FilteredSearchContext extends SearchContext { } @Override - public SearchContext trackTotalHits(boolean trackTotalHits) { - return in.trackTotalHits(trackTotalHits); + public SearchContext trackTotalHitsUpTo(int trackTotalHitsUpTo) { + return in.trackTotalHitsUpTo(trackTotalHitsUpTo); } @Override - public boolean trackTotalHits() { - return in.trackTotalHits(); + public int trackTotalHitsUpTo() { + return in.trackTotalHitsUpTo(); } @Override diff --git a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java index 48ab4914e38..e78ce7f3fb1 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java +++ b/server/src/main/java/org/elasticsearch/search/internal/InternalSearchResponse.java @@ -35,9 +35,12 @@ import java.io.IOException; * {@link SearchResponseSections} subclass that can be serialized over the wire. */ public class InternalSearchResponse extends SearchResponseSections implements Writeable, ToXContentFragment { - public static InternalSearchResponse empty() { - return new InternalSearchResponse(SearchHits.empty(), null, null, null, false, null, 1); + return empty(true); + } + + public static InternalSearchResponse empty(boolean withTotalHits) { + return new InternalSearchResponse(SearchHits.empty(withTotalHits), null, null, null, false, null, 1); } public InternalSearchResponse(SearchHits hits, InternalAggregations aggregations, Suggest suggest, diff --git a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java index 70a52c39ee1..768143dd8fb 100644 --- a/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java +++ b/server/src/main/java/org/elasticsearch/search/internal/SearchContext.java @@ -82,6 +82,10 @@ import java.util.concurrent.atomic.AtomicBoolean; public abstract class SearchContext extends AbstractRefCounted implements Releasable { public static final int DEFAULT_TERMINATE_AFTER = 0; + public static final int TRACK_TOTAL_HITS_ACCURATE = Integer.MAX_VALUE; + public static final int TRACK_TOTAL_HITS_DISABLED = -1; + public static final int DEFAULT_TRACK_TOTAL_HITS_UP_TO = TRACK_TOTAL_HITS_ACCURATE; + private Map> clearables = null; private final AtomicBoolean closed = new AtomicBoolean(false); private InnerHitsContext innerHitsContext; @@ -240,12 +244,13 @@ public abstract class SearchContext extends AbstractRefCounted implements Releas public abstract boolean trackScores(); - public abstract SearchContext trackTotalHits(boolean trackTotalHits); + public abstract SearchContext trackTotalHitsUpTo(int trackTotalHits); /** - * Indicates if the total hit count for the query should be tracked. Defaults to {@code true} + * Indicates the total number of hits to count accurately. + * Defaults to {@link #DEFAULT_TRACK_TOTAL_HITS_UP_TO}. */ - public abstract boolean trackTotalHits(); + public abstract int trackTotalHitsUpTo(); public abstract SearchContext searchAfter(FieldDoc searchAfter); diff --git a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java index 10fc6a648af..ce27f03daea 100644 --- a/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java +++ b/server/src/main/java/org/elasticsearch/search/query/QueryPhase.java @@ -166,7 +166,7 @@ public class QueryPhase implements SearchPhase { } // ... and stop collecting after ${size} matches searchContext.terminateAfter(searchContext.size()); - searchContext.trackTotalHits(false); + searchContext.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_DISABLED); } else if (canEarlyTerminate(reader, searchContext.sort())) { // now this gets interesting: since the search sort is a prefix of the index sort, we can directly // skip to the desired doc @@ -177,7 +177,7 @@ public class QueryPhase implements SearchPhase { .build(); query = bq; } - searchContext.trackTotalHits(false); + searchContext.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_DISABLED); } } } diff --git a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java index fcf70a4f98c..44c1b35ad14 100644 --- a/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java +++ b/server/src/main/java/org/elasticsearch/search/query/TopDocsCollectorContext.java @@ -92,27 +92,32 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { * Ctr * @param reader The index reader * @param query The query to execute - * @param trackTotalHits True if the total number of hits should be tracked + * @param trackTotalHitsUpTo True if the total number of hits should be tracked * @param hasFilterCollector True if the collector chain contains a filter */ private EmptyTopDocsCollectorContext(IndexReader reader, Query query, - boolean trackTotalHits, boolean hasFilterCollector) throws IOException { + int trackTotalHitsUpTo, boolean hasFilterCollector) throws IOException { super(REASON_SEARCH_COUNT, 0); - if (trackTotalHits) { + if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_DISABLED) { + this.collector = new EarlyTerminatingCollector(new TotalHitCountCollector(), 0, false); + // for bwc hit count is set to 0, it will be converted to -1 by the coordinating node + this.hitCountSupplier = () -> new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO); + } else { TotalHitCountCollector hitCountCollector = new TotalHitCountCollector(); // implicit total hit counts are valid only when there is no filter collector in the chain int hitCount = hasFilterCollector ? -1 : shortcutTotalHitCount(reader, query); if (hitCount == -1) { - this.collector = hitCountCollector; - this.hitCountSupplier = () -> new TotalHits(hitCountCollector.getTotalHits(), TotalHits.Relation.EQUAL_TO); + if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_ACCURATE) { + this.collector = hitCountCollector; + this.hitCountSupplier = () -> new TotalHits(hitCountCollector.getTotalHits(), TotalHits.Relation.EQUAL_TO); + } else { + this.collector = new EarlyTerminatingCollector(hitCountCollector, trackTotalHitsUpTo, false); + this.hitCountSupplier = () -> new TotalHits(hitCount, TotalHits.Relation.EQUAL_TO); + } } else { this.collector = new EarlyTerminatingCollector(hitCountCollector, 0, false); this.hitCountSupplier = () -> new TotalHits(hitCount, TotalHits.Relation.EQUAL_TO); } - } else { - this.collector = new EarlyTerminatingCollector(new TotalHitCountCollector(), 0, false); - // for bwc hit count is set to 0, it will be converted to -1 by the coordinating node - this.hitCountSupplier = () -> new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO); } } @@ -184,11 +189,11 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { } } - private final @Nullable SortAndFormats sortAndFormats; private final Collector collector; - private final Supplier totalHitsSupplier; - private final Supplier topDocsSupplier; - private final Supplier maxScoreSupplier; + protected final @Nullable SortAndFormats sortAndFormats; + protected final Supplier totalHitsSupplier; + protected final Supplier topDocsSupplier; + protected final Supplier maxScoreSupplier; /** * Ctr @@ -198,7 +203,7 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { * @param numHits The number of top hits to retrieve * @param searchAfter The doc this request should "search after" * @param trackMaxScore True if max score should be tracked - * @param trackTotalHits True if the total number of hits should be tracked + * @param trackTotalHitsUpTo True if the total number of hits should be tracked * @param hasFilterCollector True if the collector chain contains at least one collector that can filters document */ private SimpleTopDocsCollectorContext(IndexReader reader, @@ -207,7 +212,7 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { @Nullable ScoreDoc searchAfter, int numHits, boolean trackMaxScore, - boolean trackTotalHits, + int trackTotalHitsUpTo, boolean hasFilterCollector) throws IOException { super(REASON_SEARCH_TOP_HITS, numHits); this.sortAndFormats = sortAndFormats; @@ -215,17 +220,20 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { // implicit total hit counts are valid only when there is no filter collector in the chain final int hitCount = hasFilterCollector ? -1 : shortcutTotalHitCount(reader, query); final TopDocsCollector topDocsCollector; - if (hitCount == -1 && trackTotalHits) { - topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, Integer.MAX_VALUE); + if (trackTotalHitsUpTo == SearchContext.TRACK_TOTAL_HITS_DISABLED) { + // don't compute hit counts via the collector + topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, 1); topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs); - totalHitsSupplier = () -> topDocsSupplier.get().totalHits; + totalHitsSupplier = () -> new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO); } else { - topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, 1); // don't compute hit counts via the collector - topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs); if (hitCount == -1) { - assert trackTotalHits == false; - totalHitsSupplier = () -> new TotalHits(0, TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO); + topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, trackTotalHitsUpTo); + topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs); + totalHitsSupplier = () -> topDocsSupplier.get().totalHits; } else { + // don't compute hit counts via the collector + topDocsCollector = createCollector(sortAndFormats, numHits, searchAfter, 1); + topDocsSupplier = new CachedSupplier<>(topDocsCollector::topDocs); totalHitsSupplier = () -> new TotalHits(hitCount, TotalHits.Relation.EQUAL_TO); } } @@ -258,7 +266,8 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { void postProcess(QuerySearchResult result) throws IOException { final TopDocs topDocs = topDocsSupplier.get(); topDocs.totalHits = totalHitsSupplier.get(); - result.topDocs(new TopDocsAndMaxScore(topDocs, maxScoreSupplier.get()), sortAndFormats == null ? null : sortAndFormats.formats); + result.topDocs(new TopDocsAndMaxScore(topDocs, maxScoreSupplier.get()), + sortAndFormats == null ? null : sortAndFormats.formats); } } @@ -273,36 +282,38 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { int numHits, boolean trackMaxScore, int numberOfShards, - boolean trackTotalHits, + int trackTotalHitsUpTo, boolean hasFilterCollector) throws IOException { super(reader, query, sortAndFormats, scrollContext.lastEmittedDoc, numHits, trackMaxScore, - trackTotalHits, hasFilterCollector); + trackTotalHitsUpTo, hasFilterCollector); this.scrollContext = Objects.requireNonNull(scrollContext); this.numberOfShards = numberOfShards; } @Override void postProcess(QuerySearchResult result) throws IOException { - super.postProcess(result); - final TopDocsAndMaxScore topDocs = result.topDocs(); + final TopDocs topDocs = topDocsSupplier.get(); + topDocs.totalHits = totalHitsSupplier.get(); + float maxScore = maxScoreSupplier.get(); if (scrollContext.totalHits == null) { // first round - scrollContext.totalHits = topDocs.topDocs.totalHits; - scrollContext.maxScore = topDocs.maxScore; + scrollContext.totalHits = topDocs.totalHits; + scrollContext.maxScore = maxScore; } else { // subsequent round: the total number of hits and // the maximum score were computed on the first round - topDocs.topDocs.totalHits = scrollContext.totalHits; - topDocs.maxScore = scrollContext.maxScore; + topDocs.totalHits = scrollContext.totalHits; + maxScore = scrollContext.maxScore; } if (numberOfShards == 1) { // if we fetch the document in the same roundtrip, we already know the last emitted doc - if (topDocs.topDocs.scoreDocs.length > 0) { + if (topDocs.scoreDocs.length > 0) { // set the last emitted doc - scrollContext.lastEmittedDoc = topDocs.topDocs.scoreDocs[topDocs.topDocs.scoreDocs.length - 1]; + scrollContext.lastEmittedDoc = topDocs.scoreDocs[topDocs.scoreDocs.length - 1]; } } - result.topDocs(topDocs, result.sortValueFormats()); + result.topDocs(new TopDocsAndMaxScore(topDocs, maxScore), + sortAndFormats == null ? null : sortAndFormats.formats); } } @@ -351,13 +362,17 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { final int totalNumDocs = Math.max(1, reader.numDocs()); if (searchContext.size() == 0) { // no matter what the value of from is - return new EmptyTopDocsCollectorContext(reader, query, searchContext.trackTotalHits(), hasFilterCollector); + return new EmptyTopDocsCollectorContext(reader, query, searchContext.trackTotalHitsUpTo(), hasFilterCollector); } else if (searchContext.scrollContext() != null) { + // we can disable the tracking of total hits after the initial scroll query + // since the total hits is preserved in the scroll context. + int trackTotalHitsUpTo = searchContext.scrollContext().totalHits != null ? + SearchContext.TRACK_TOTAL_HITS_DISABLED : searchContext.trackTotalHitsUpTo(); // no matter what the value of from is int numDocs = Math.min(searchContext.size(), totalNumDocs); return new ScrollingTopDocsCollectorContext(reader, query, searchContext.scrollContext(), searchContext.sort(), numDocs, searchContext.trackScores(), searchContext.numberOfShards(), - searchContext.trackTotalHits(), hasFilterCollector); + trackTotalHitsUpTo, hasFilterCollector); } else if (searchContext.collapse() != null) { boolean trackScores = searchContext.sort() == null ? true : searchContext.trackScores(); int numDocs = Math.min(searchContext.from() + searchContext.size(), totalNumDocs); @@ -372,7 +387,7 @@ abstract class TopDocsCollectorContext extends QueryCollectorContext { } } return new SimpleTopDocsCollectorContext(reader, query, searchContext.sort(), searchContext.searchAfter(), numDocs, - searchContext.trackScores(), searchContext.trackTotalHits(), hasFilterCollector) { + searchContext.trackScores(), searchContext.trackTotalHitsUpTo(), hasFilterCollector) { @Override boolean shouldRescore() { return rescore; diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java index 3859e3b7f38..a5ab81d83fb 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchPhaseControllerTests.java @@ -49,6 +49,7 @@ import org.elasticsearch.search.aggregations.metrics.InternalMax; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.FetchSearchResult; import org.elasticsearch.search.internal.InternalSearchResponse; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.query.QuerySearchResult; import org.elasticsearch.search.suggest.Suggest; import org.elasticsearch.search.suggest.completion.CompletionSuggestion; @@ -162,16 +163,18 @@ public class SearchPhaseControllerTests extends ESTestCase { int nShards = randomIntBetween(1, 20); int queryResultSize = randomBoolean() ? 0 : randomIntBetween(1, nShards * 2); AtomicArray queryResults = generateQueryResults(nShards, suggestions, queryResultSize, false); - for (boolean trackTotalHits : new boolean[] {true, false}) { + for (int trackTotalHits : new int[] {SearchContext.TRACK_TOTAL_HITS_DISABLED, SearchContext.TRACK_TOTAL_HITS_ACCURATE}) { SearchPhaseController.ReducedQueryPhase reducedQueryPhase = searchPhaseController.reducedQueryPhase(queryResults.asList(), false, trackTotalHits, true); AtomicArray fetchResults = generateFetchResults(nShards, reducedQueryPhase.sortedTopDocs.scoreDocs, reducedQueryPhase.suggest); InternalSearchResponse mergedResponse = searchPhaseController.merge(false, - reducedQueryPhase, - fetchResults.asList(), fetchResults::get); - if (trackTotalHits == false) { + reducedQueryPhase, fetchResults.asList(), fetchResults::get); + if (trackTotalHits == SearchContext.TRACK_TOTAL_HITS_DISABLED) { assertNull(mergedResponse.hits.getTotalHits()); + } else { + assertThat(mergedResponse.hits.getTotalHits().value, equalTo(0L)); + assertEquals(mergedResponse.hits.getTotalHits().relation, Relation.EQUAL_TO); } for (SearchHit hit : mergedResponse.hits().getHits()) { SearchPhaseResult searchPhaseResult = fetchResults.get(hit.getShard().getShardId().id()); diff --git a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java index 719a14491ae..ed32c0a1dd3 100644 --- a/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java +++ b/server/src/test/java/org/elasticsearch/action/search/SearchRequestTests.java @@ -78,6 +78,7 @@ public class SearchRequestTests extends AbstractSearchTestCase { public void testReadFromPre7_0_0() throws IOException { String msg = "AAEBBWluZGV4AAAAAQACAAAA/////w8AAAAAAAAA/////w8AAAAAAAACAAAAAAABAAMCBAUBAAKABACAAQIAAA=="; try (StreamInput in = StreamInput.wrap(Base64.getDecoder().decode(msg))) { + in.setVersion(Version.V_6_6_0); SearchRequest searchRequest = new SearchRequest(in); assertArrayEquals(new String[]{"index"}, searchRequest.indices()); assertNull(searchRequest.getLocalClusterAlias()); diff --git a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java index 7e9c0153b72..a321ff9c1a8 100644 --- a/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java +++ b/server/src/test/java/org/elasticsearch/search/query/QueryPhaseTests.java @@ -57,6 +57,7 @@ import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.IndexShardTestCase; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.internal.ScrollContext; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.sort.SortAndFormats; import org.elasticsearch.test.TestSearchContext; @@ -453,7 +454,7 @@ public class QueryPhaseTests extends IndexShardTestCase { { contextSearcher = getAssertingEarlyTerminationSearcher(reader, 1); - context.trackTotalHits(false); + context.trackTotalHitsUpTo(SearchContext.TRACK_TOTAL_HITS_DISABLED); QueryPhase.execute(context, contextSearcher, checkCancelled -> {}); assertNull(context.queryResult().terminatedEarly()); assertThat(context.queryResult().topDocs().topDocs.scoreDocs.length, equalTo(1)); diff --git a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java index 279bddebc4a..5d96cd37b05 100644 --- a/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java +++ b/test/framework/src/main/java/org/elasticsearch/search/RandomSearchRequestGenerator.java @@ -39,6 +39,7 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.collapse.CollapseBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; +import org.elasticsearch.search.internal.SearchContext; import org.elasticsearch.search.rescore.RescorerBuilder; import org.elasticsearch.search.searchafter.SearchAfterBuilder; import org.elasticsearch.search.slice.SliceBuilder; @@ -157,7 +158,13 @@ public class RandomSearchRequestGenerator { builder.terminateAfter(randomIntBetween(1, 100000)); } if (randomBoolean()) { - builder.trackTotalHits(randomBoolean()); + if (randomBoolean()) { + builder.trackTotalHits(randomBoolean()); + } else { + builder.trackTotalHitsUpTo( + randomIntBetween(SearchContext.TRACK_TOTAL_HITS_DISABLED, SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO) + ); + } } switch(randomInt(2)) { diff --git a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java index 9d033835616..18edb5ec379 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java +++ b/test/framework/src/main/java/org/elasticsearch/test/TestSearchContext.java @@ -83,7 +83,7 @@ public class TestSearchContext extends SearchContext { SearchTask task; SortAndFormats sort; boolean trackScores = false; - boolean trackTotalHits = true; + int trackTotalHitsUpTo = SearchContext.DEFAULT_TRACK_TOTAL_HITS_UP_TO; ContextIndexSearcher searcher; int size; @@ -364,14 +364,14 @@ public class TestSearchContext extends SearchContext { } @Override - public SearchContext trackTotalHits(boolean trackTotalHits) { - this.trackTotalHits = trackTotalHits; + public SearchContext trackTotalHitsUpTo(int trackTotalHitsUpTo) { + this.trackTotalHitsUpTo = trackTotalHitsUpTo; return this; } @Override - public boolean trackTotalHits() { - return trackTotalHits; + public int trackTotalHitsUpTo() { + return trackTotalHitsUpTo; } @Override diff --git a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java index 6c3b99d8da3..25fbef7ada7 100644 --- a/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java +++ b/x-pack/plugin/ccr/qa/src/main/java/org/elasticsearch/xpack/ccr/ESCCRRestTestCase.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Map; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -99,7 +99,7 @@ public class ESCCRRestTestCase extends ESRestTestCase { request.addParameter("size", Integer.toString(expectedNumDocs)); request.addParameter("sort", "field:asc"); request.addParameter("q", query); - request.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + request.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); Map response = toMap(client.performRequest(request)); int numDocs = (int) XContentMapValues.extractValue("hits.total", response); diff --git a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java index a027bf06f40..1d9960e711f 100644 --- a/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java +++ b/x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java @@ -25,7 +25,7 @@ public class RestRollupSearchAction extends BaseRestHandler { private static final Set RESPONSE_PARAMS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList( RestSearchAction.TYPED_KEYS_PARAM, - RestSearchAction.TOTAL_HIT_AS_INT_PARAM))); + RestSearchAction.TOTAL_HITS_AS_INT_PARAM))); public RestRollupSearchAction(Settings settings, RestController controller) { super(settings); @@ -40,6 +40,7 @@ public class RestRollupSearchAction extends BaseRestHandler { SearchRequest searchRequest = new SearchRequest(); restRequest.withContentOrSourceParamParserOrNull(parser -> RestSearchAction.parseSearchRequest(searchRequest, restRequest, parser, size -> searchRequest.source().size(size))); + RestSearchAction.checkRestTotalHits(restRequest, searchRequest); return channel -> client.execute(RollupSearchAction.INSTANCE, searchRequest, new RestToXContentListener<>(channel)); } diff --git a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java index 413447dd204..57c94478848 100644 --- a/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java +++ b/x-pack/plugin/src/test/java/org/elasticsearch/xpack/test/rest/XPackRestIT.java @@ -46,7 +46,7 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singletonList; import static java.util.Collections.singletonMap; import static org.elasticsearch.common.xcontent.support.XContentMapValues.extractValue; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.hasSize; import static org.hamcrest.Matchers.is; @@ -139,7 +139,7 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase { return; } Request searchWatchesRequest = new Request("GET", ".watches/_search"); - searchWatchesRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchWatchesRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); searchWatchesRequest.addParameter("size", "1000"); Response response = adminClient().performRequest(searchWatchesRequest); ObjectPath objectPathResponse = ObjectPath.createFromResponse(response); @@ -184,7 +184,7 @@ public class XPackRestIT extends ESClientYamlSuiteTestCase { () -> "Exception when enabling monitoring"); Map searchParams = new HashMap<>(); searchParams.put("index", ".monitoring-*"); - searchParams.put(TOTAL_HIT_AS_INT_PARAM, "true"); + searchParams.put(TOTAL_HITS_AS_INT_PARAM, "true"); awaitCallApi("search", searchParams, emptyList(), response -> ((Number) response.evaluate("hits.total")).intValue() > 0, () -> "Exception when waiting for monitoring documents to be indexed"); diff --git a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java index 88238ae4601..e52a6dd3b43 100644 --- a/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java +++ b/x-pack/qa/full-cluster-restart/src/test/java/org/elasticsearch/xpack/restart/FullClusterRestartIT.java @@ -42,7 +42,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.elasticsearch.common.unit.TimeValue.timeValueSeconds; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -346,7 +346,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { client().performRequest(new Request("POST", "id-test-results-rollup/_refresh")); final Request searchRequest = new Request("GET", "id-test-results-rollup/_search"); if (isRunningAgainstOldCluster() == false) { - searchRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); } try { Map searchResponse = entityAsMap(client().performRequest(searchRequest)); @@ -389,7 +389,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { client().performRequest(new Request("POST", "id-test-results-rollup/_refresh")); final Request searchRequest = new Request("GET", "id-test-results-rollup/_search"); if (isRunningAgainstOldCluster() == false) { - searchRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); } try { Map searchResponse = entityAsMap(client().performRequest(searchRequest)); @@ -500,7 +500,7 @@ public class FullClusterRestartIT extends AbstractFullClusterRestartTestCase { assertThat(basic, hasEntry(is("password"), anyOf(startsWith("::es_encrypted::"), is("::es_redacted::")))); Request searchRequest = new Request("GET", ".watcher-history*/_search"); if (isRunningAgainstOldCluster() == false) { - searchRequest.addParameter(RestSearchAction.TOTAL_HIT_AS_INT_PARAM, "true"); + searchRequest.addParameter(RestSearchAction.TOTAL_HITS_AS_INT_PARAM, "true"); } Map history = entityAsMap(client().performRequest(searchRequest)); Map hits = (Map) history.get("hits"); diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java index d657a42d7df..b080a77e064 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/IndexingIT.java @@ -17,7 +17,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.hamcrest.Matchers.equalTo; /** @@ -143,7 +143,7 @@ public class IndexingIT extends AbstractUpgradeTestCase { private void assertCount(String index, int count) throws IOException { Request searchTestIndexRequest = new Request("POST", "/" + index + "/_search"); - searchTestIndexRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchTestIndexRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); searchTestIndexRequest.addParameter("filter_path", "hits.total"); Response searchTestIndexResponse = client().performRequest(searchTestIndexRequest); assertEquals("{\"hits\":{\"total\":" + count + "}}", diff --git a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java index 9eef5968f27..2f653c2bbf1 100644 --- a/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java +++ b/x-pack/qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RollupIDUpgradeIT.java @@ -25,7 +25,7 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.equalTo; @@ -197,7 +197,7 @@ public class RollupIDUpgradeIT extends AbstractUpgradeTestCase { collectedIDs.clear(); client().performRequest(new Request("POST", "rollup/_refresh")); final Request searchRequest = new Request("GET", "rollup/_search"); - searchRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); try { Map searchResponse = entityAsMap(client().performRequest(searchRequest)); assertNotNull(ObjectPath.eval("hits.total", searchResponse)); diff --git a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java index 2341878d924..226ebc29ec7 100644 --- a/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java +++ b/x-pack/qa/smoke-test-watcher-with-security/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherWithSecurityIT.java @@ -25,7 +25,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.greaterThanOrEqualTo; @@ -324,7 +324,7 @@ public class SmokeTestWatcherWithSecurityIT extends ESRestTestCase { builder.endObject(); Request searchRequest = new Request("POST", "/.watcher-history-*/_search"); - searchRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); searchRequest.setJsonEntity(Strings.toString(builder)); Response response = client().performRequest(searchRequest); ObjectPath objectPath = ObjectPath.createFromResponse(response); diff --git a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java index 162c6c07643..a7350fcff03 100644 --- a/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java +++ b/x-pack/qa/smoke-test-watcher/src/test/java/org/elasticsearch/smoketest/SmokeTestWatcherTestSuiteIT.java @@ -23,7 +23,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicReference; import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HIT_AS_INT_PARAM; +import static org.elasticsearch.rest.action.search.RestSearchAction.TOTAL_HITS_AS_INT_PARAM; import static org.elasticsearch.xpack.core.security.authc.support.UsernamePasswordToken.basicAuthHeaderValue; import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.hasEntry; @@ -194,7 +194,7 @@ public class SmokeTestWatcherTestSuiteIT extends ESRestTestCase { builder.endObject(); Request searchRequest = new Request("POST", "/.watcher-history-*/_search"); - searchRequest.addParameter(TOTAL_HIT_AS_INT_PARAM, "true"); + searchRequest.addParameter(TOTAL_HITS_AS_INT_PARAM, "true"); searchRequest.setJsonEntity(Strings.toString(builder)); Response response = client().performRequest(searchRequest); ObjectPath objectPath = ObjectPath.createFromResponse(response);