From 34677b9c83f967011440c018b7676b3701c26649 Mon Sep 17 00:00:00 2001 From: Milan Mrdjen Date: Fri, 2 Nov 2018 14:21:19 +0100 Subject: [PATCH] HLRC: Add document _count API (#34267) Add `count()` api method, `CountRequest` and `CountResponse` classes to HLRC. Code in server module is unchanged. Relates to #27205 --- .../client/RequestConverters.java | 11 + .../client/RestHighLevelClient.java | 27 ++ .../client/core/CountRequest.java | 206 +++++++++++++++ .../client/core/CountResponse.java | 236 ++++++++++++++++++ .../client/RequestConvertersTests.java | 67 +++++ .../client/RestHighLevelClientTests.java | 1 - .../org/elasticsearch/client/SearchIT.java | 67 +++++ .../client/core/CountRequestTests.java | 95 +++++++ .../client/core/CountResponseTests.java | 126 ++++++++++ .../documentation/SearchDocumentationIT.java | 122 +++++++++ .../high-level/search/count.asciidoc | 114 +++++++++ .../high-level/supported-apis.asciidoc | 2 + 12 files changed, 1073 insertions(+), 1 deletion(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java create mode 100644 docs/java-rest/high-level/search/count.asciidoc diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 39bc03f4117..38dbbb8f151 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -49,6 +49,7 @@ import org.elasticsearch.action.support.ActiveShardCount; import org.elasticsearch.action.support.IndicesOptions; import org.elasticsearch.action.support.WriteRequest; import org.elasticsearch.action.update.UpdateRequest; +import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.client.security.RefreshPolicy; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Nullable; @@ -442,6 +443,16 @@ final class RequestConverters { return request; } + static Request count(CountRequest countRequest) throws IOException { + Request request = new Request(HttpPost.METHOD_NAME, endpoint(countRequest.indices(), countRequest.types(), "_count")); + Params params = new Params(request); + params.withRouting(countRequest.routing()); + params.withPreference(countRequest.preference()); + params.withIndicesOptions(countRequest.indicesOptions()); + request.setEntity(createEntity(countRequest.source(), REQUEST_BODY_CONTENT_TYPE)); + return request; + } + static Request explain(ExplainRequest explainRequest) throws IOException { Request request = new Request(HttpGet.METHOD_NAME, endpoint(explainRequest.index(), explainRequest.type(), explainRequest.id(), "_explain")); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index d8dce990a4d..11fff4c0a6b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -56,6 +56,8 @@ import org.elasticsearch.action.search.SearchScrollRequest; import org.elasticsearch.action.support.master.AcknowledgedResponse; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.action.update.UpdateResponse; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.client.core.TermVectorsResponse; import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.common.CheckedConsumer; @@ -791,6 +793,31 @@ public class RestHighLevelClient implements Closeable { emptySet()); } + /** + * Executes a count request using the Count API. + * See Count API on elastic.co + * @param countRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + * @throws IOException in case there is a problem sending the request or parsing back the response + */ + public final CountResponse count(CountRequest countRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(countRequest, RequestConverters::count, options, CountResponse::fromXContent, + emptySet()); + } + + /** + * Asynchronously executes a count request using the Count API. + * See Count API on elastic.co + * @param countRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + */ + public final void countAsync(CountRequest countRequest, RequestOptions options, ActionListener listener) { + performRequestAsyncAndParseEntity(countRequest, RequestConverters::count, options,CountResponse::fromXContent, + listener, emptySet()); + } + /** * Updates a document using the Update API. * See Update API on elastic.co diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java new file mode 100644 index 00000000000..6d4589c7861 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountRequest.java @@ -0,0 +1,206 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.core; + +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.Strings; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.util.Arrays; +import java.util.Objects; + +import static org.elasticsearch.action.search.SearchRequest.DEFAULT_INDICES_OPTIONS; + +/** + * Encapsulates a request to _count API against one, several or all indices. + */ +public final class CountRequest extends ActionRequest implements IndicesRequest.Replaceable { + + private String[] indices = Strings.EMPTY_ARRAY; + private String[] types = Strings.EMPTY_ARRAY; + private String routing; + private String preference; + private SearchSourceBuilder searchSourceBuilder; + private IndicesOptions indicesOptions = DEFAULT_INDICES_OPTIONS; + + public CountRequest() { + this.searchSourceBuilder = new SearchSourceBuilder(); + } + + /** + * Constructs a new count request against the indices. No indices provided here means that count will execute on all indices. + */ + public CountRequest(String... indices) { + this(indices, new SearchSourceBuilder()); + } + + /** + * Constructs a new search request against the provided indices with the given search source. + */ + public CountRequest(String[] indices, SearchSourceBuilder searchSourceBuilder) { + indices(indices); + this.searchSourceBuilder = searchSourceBuilder; + } + + @Override + public ActionRequestValidationException validate() { + return null; + } + + /** + * Sets the indices the count will be executed on. + */ + public CountRequest indices(String... indices) { + Objects.requireNonNull(indices, "indices must not be null"); + for (String index : indices) { + Objects.requireNonNull(index, "index must not be null"); + } + this.indices = indices; + return this; + } + + /** + * The source of the count request. + */ + public CountRequest source(SearchSourceBuilder searchSourceBuilder) { + this.searchSourceBuilder = Objects.requireNonNull(searchSourceBuilder, "source must not be null"); + return this; + } + + /** + * The document types to execute the count against. Defaults to be executed against all types. + * + * @deprecated Types are going away, prefer filtering on a type. + */ + @Deprecated + public CountRequest types(String... types) { + Objects.requireNonNull(types, "types must not be null"); + for (String type : types) { + Objects.requireNonNull(type, "type must not be null"); + } + this.types = types; + return this; + } + + /** + * The routing values to control the shards that the search will be executed on. + */ + public CountRequest routing(String routing) { + this.routing = routing; + return this; + } + + /** + * A comma separated list of routing values to control the shards the count will be executed on. + */ + public CountRequest routing(String... routings) { + this.routing = Strings.arrayToCommaDelimitedString(routings); + return this; + } + + /** + * Returns the indices options used to resolve indices. They tell for instance whether a single index is accepted, whether an empty + * array will be converted to _all, and how wildcards will be expanded if needed. + * + * @see org.elasticsearch.action.support.IndicesOptions + */ + public CountRequest indicesOptions(IndicesOptions indicesOptions) { + this.indicesOptions = Objects.requireNonNull(indicesOptions, "indicesOptions must not be null"); + return this; + } + + /** + * Sets the preference to execute the count. Defaults to randomize across shards. Can be set to {@code _local} to prefer local shards + * or a custom value, which guarantees that the same order will be used across different requests. + */ + public CountRequest preference(String preference) { + this.preference = preference; + return this; + } + + public IndicesOptions indicesOptions() { + return this.indicesOptions; + } + + public String routing() { + return this.routing; + } + + public String preference() { + return this.preference; + } + + public String[] indices() { + return Arrays.copyOf(this.indices, this.indices.length); + } + + public Float minScore() { + return this.searchSourceBuilder.minScore(); + } + + public CountRequest minScore(Float minScore) { + this.searchSourceBuilder.minScore(minScore); + return this; + } + + public int terminateAfter() { + return this.searchSourceBuilder.terminateAfter(); + } + + public CountRequest terminateAfter(int terminateAfter) { + this.searchSourceBuilder.terminateAfter(terminateAfter); + return this; + } + + public String[] types() { + return Arrays.copyOf(this.types, this.types.length); + } + + public SearchSourceBuilder source() { + return this.searchSourceBuilder; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + CountRequest that = (CountRequest) o; + return Objects.equals(indicesOptions, that.indicesOptions) && + Arrays.equals(indices, that.indices) && + Arrays.equals(types, that.types) && + Objects.equals(routing, that.routing) && + Objects.equals(preference, that.preference); + } + + @Override + public int hashCode() { + int result = Objects.hash(indicesOptions, routing, preference); + result = 31 * result + Arrays.hashCode(indices); + result = 31 * result + Arrays.hashCode(types); + return result; + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java new file mode 100644 index 00000000000..f97f79127e6 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/core/CountResponse.java @@ -0,0 +1,236 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.core; + +import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.RestStatus; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken; + +/** + * A response to _count API request. + */ +public final class CountResponse extends ActionResponse { + + static final ParseField COUNT = new ParseField("count"); + static final ParseField TERMINATED_EARLY = new ParseField("terminated_early"); + static final ParseField SHARDS = new ParseField("_shards"); + + private final long count; + private final Boolean terminatedEarly; + private final ShardStats shardStats; + + public CountResponse(long count, Boolean terminatedEarly, ShardStats shardStats) { + this.count = count; + this.terminatedEarly = terminatedEarly; + this.shardStats = shardStats; + } + + public ShardStats getShardStats() { + return shardStats; + } + + /** + * Number of documents matching request. + */ + public long getCount() { + return count; + } + + /** + * The total number of shards the search was executed on. + */ + public int getTotalShards() { + return shardStats.totalShards; + } + + /** + * The successful number of shards the search was executed on. + */ + public int getSuccessfulShards() { + return shardStats.successfulShards; + } + + /** + * The number of shards skipped due to pre-filtering + */ + public int getSkippedShards() { + return shardStats.skippedShards; + } + + /** + * The failed number of shards the search was executed on. + */ + public int getFailedShards() { + return shardStats.shardFailures.length; + } + + /** + * The failures that occurred during the search. + */ + public ShardSearchFailure[] getShardFailures() { + return shardStats.shardFailures; + } + + public RestStatus status() { + return RestStatus.status(shardStats.successfulShards, shardStats.totalShards, shardStats.shardFailures); + } + + public static CountResponse fromXContent(XContentParser parser) throws IOException { + ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation); + parser.nextToken(); + ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.currentToken(), parser::getTokenLocation); + String currentName = parser.currentName(); + Boolean terminatedEarly = null; + long count = 0; + ShardStats shardStats = new ShardStats(-1, -1,0, ShardSearchFailure.EMPTY_ARRAY); + + for (XContentParser.Token token = parser.nextToken(); token != XContentParser.Token.END_OBJECT; token = parser.nextToken()) { + if (token == XContentParser.Token.FIELD_NAME) { + currentName = parser.currentName(); + } else if (token.isValue()) { + if (COUNT.match(currentName, parser.getDeprecationHandler())) { + count = parser.longValue(); + } else if (TERMINATED_EARLY.match(currentName, parser.getDeprecationHandler())) { + terminatedEarly = parser.booleanValue(); + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_OBJECT) { + if (SHARDS.match(currentName, parser.getDeprecationHandler())) { + shardStats = ShardStats.fromXContent(parser); + } else { + parser.skipChildren(); + } + } + } + return new CountResponse(count, terminatedEarly, shardStats); + } + + @Override + public String toString() { + String s = "{" + + "count=" + count + + (isTerminatedEarly() != null ? ", terminatedEarly=" + terminatedEarly : "") + + ", " + shardStats + + '}'; + return s; + } + + public Boolean isTerminatedEarly() { + return terminatedEarly; + } + + /** + * Encapsulates _shards section of count api response. + */ + public static final class ShardStats { + + static final ParseField FAILED = new ParseField("failed"); + static final ParseField SKIPPED = new ParseField("skipped"); + static final ParseField TOTAL = new ParseField("total"); + static final ParseField SUCCESSFUL = new ParseField("successful"); + static final ParseField FAILURES = new ParseField("failures"); + + private final int successfulShards; + private final int totalShards; + private final int skippedShards; + private final ShardSearchFailure[] shardFailures; + + public ShardStats(int successfulShards, int totalShards, int skippedShards, ShardSearchFailure[] shardFailures) { + this.successfulShards = successfulShards; + this.totalShards = totalShards; + this.skippedShards = skippedShards; + this.shardFailures = Arrays.copyOf(shardFailures, shardFailures.length); + } + + public int getSuccessfulShards() { + return successfulShards; + } + + public int getTotalShards() { + return totalShards; + } + + public int getSkippedShards() { + return skippedShards; + } + + public ShardSearchFailure[] getShardFailures() { + return Arrays.copyOf(shardFailures, shardFailures.length, ShardSearchFailure[].class); + } + + static ShardStats fromXContent(XContentParser parser) throws IOException { + int successfulShards = -1; + int totalShards = -1; + int skippedShards = 0; //BWC @see org.elasticsearch.action.search.SearchResponse + List failures = new ArrayList<>(); + XContentParser.Token token; + String currentName = parser.currentName(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentName = parser.currentName(); + } else if (token.isValue()) { + if (FAILED.match(currentName, parser.getDeprecationHandler())) { + parser.intValue(); + } else if (SKIPPED.match(currentName, parser.getDeprecationHandler())) { + skippedShards = parser.intValue(); + } else if (TOTAL.match(currentName, parser.getDeprecationHandler())) { + totalShards = parser.intValue(); + } else if (SUCCESSFUL.match(currentName, parser.getDeprecationHandler())) { + successfulShards = parser.intValue(); + } else { + parser.skipChildren(); + } + } else if (token == XContentParser.Token.START_ARRAY) { + if (FAILURES.match(currentName, parser.getDeprecationHandler())) { + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + failures.add(ShardSearchFailure.fromXContent(parser)); + } + } else { + parser.skipChildren(); + } + } else { + parser.skipChildren(); + } + } + return new ShardStats(successfulShards, totalShards, skippedShards, failures.toArray(new ShardSearchFailure[failures.size()])); + } + + @Override + public String toString() { + return "_shards : {" + + "total=" + totalShards + + ", successful=" + successfulShards + + ", skipped=" + skippedShards + + ", failed=" + (shardFailures != null && shardFailures.length > 0 ? shardFailures.length : 0 ) + + (shardFailures != null && shardFailures.length > 0 ? ", failures: " + Arrays.asList(shardFailures): "") + + '}'; + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java index 76b64241ef6..1e0f0f70b2f 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java @@ -56,6 +56,7 @@ import org.elasticsearch.action.support.replication.ReplicationRequest; import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.action.update.UpdateRequest; import org.elasticsearch.client.RequestConverters.EndpointBuilder; +import org.elasticsearch.client.core.CountRequest; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesArray; @@ -968,6 +969,72 @@ public class RequestConvertersTests extends ESTestCase { expectThrows(NullPointerException.class, () -> new SearchRequest().types((String[]) null)); } + public void testCountNotNullSource() throws IOException { + //as we create SearchSourceBuilder in CountRequest constructor + CountRequest countRequest = new CountRequest(); + Request request = RequestConverters.count(countRequest); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals("/_count", request.getEndpoint()); + assertNotNull(request.getEntity()); + } + + public void testCount() throws Exception { + String[] indices = randomIndicesNames(0, 5); + CountRequest countRequest = new CountRequest(indices); + + int numTypes = randomIntBetween(0, 5); + String[] types = new String[numTypes]; + for (int i = 0; i < numTypes; i++) { + types[i] = "type-" + randomAlphaOfLengthBetween(2, 5); + } + countRequest.types(types); + + Map expectedParams = new HashMap<>(); + setRandomCountParams(countRequest, expectedParams); + setRandomIndicesOptions(countRequest::indicesOptions, countRequest::indicesOptions, expectedParams); + + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + if (frequently()) { + if (randomBoolean()) { + searchSourceBuilder.minScore(randomFloat()); + } + } + countRequest.source(searchSourceBuilder); + Request request = RequestConverters.count(countRequest); + StringJoiner endpoint = new StringJoiner("/", "/", ""); + String index = String.join(",", indices); + if (Strings.hasLength(index)) { + endpoint.add(index); + } + String type = String.join(",", types); + if (Strings.hasLength(type)) { + endpoint.add(type); + } + endpoint.add("_count"); + assertEquals(HttpPost.METHOD_NAME, request.getMethod()); + assertEquals(endpoint.toString(), request.getEndpoint()); + assertEquals(expectedParams, request.getParameters()); + assertToXContentBody(searchSourceBuilder, request.getEntity()); + } + + public void testCountNullIndicesAndTypes() { + expectThrows(NullPointerException.class, () -> new CountRequest((String[]) null)); + expectThrows(NullPointerException.class, () -> new CountRequest().indices((String[]) null)); + expectThrows(NullPointerException.class, () -> new CountRequest().types((String[]) null)); + } + + private static void setRandomCountParams(CountRequest countRequest, + Map expectedParams) { + if (randomBoolean()) { + countRequest.routing(randomAlphaOfLengthBetween(3, 10)); + expectedParams.put("routing", countRequest.routing()); + } + if (randomBoolean()) { + countRequest.preference(randomAlphaOfLengthBetween(3, 10)); + expectedParams.put("preference", countRequest.preference()); + } + } + public void testMultiSearch() throws IOException { int numberOfSearchRequests = randomIntBetween(0, 32); MultiSearchRequest multiSearchRequest = new MultiSearchRequest(); diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java index d6c77d6f2a5..38810285a5d 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java @@ -662,7 +662,6 @@ public class RestHighLevelClientTests extends ESTestCase { //this list should be empty once the high-level client is feature complete String[] notYetSupportedApi = new String[]{ "cluster.remote_info", - "count", "create", "get_source", "indices.delete_alias", diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java index f6aa97def28..e6a2f5d6178 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/SearchIT.java @@ -35,6 +35,8 @@ import org.elasticsearch.action.search.MultiSearchResponse; import org.elasticsearch.action.search.SearchRequest; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchScrollRequest; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.unit.TimeValue; @@ -1233,4 +1235,69 @@ public class SearchIT extends ESRestHighLevelClientTestCase { assertEquals(0, searchResponse.getShardFailures().length); assertEquals(SearchResponse.Clusters.EMPTY, searchResponse.getClusters()); } + + public void testCountOneIndexNoQuery() throws IOException { + CountRequest countRequest = new CountRequest("index"); + CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync); + assertCountHeader(countResponse); + assertEquals(5, countResponse.getCount()); + } + + public void testCountMultipleIndicesNoQuery() throws IOException { + CountRequest countRequest = new CountRequest("index", "index1"); + CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync); + assertCountHeader(countResponse); + assertEquals(7, countResponse.getCount()); + } + + public void testCountAllIndicesNoQuery() throws IOException { + CountRequest countRequest = new CountRequest(); + CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync); + assertCountHeader(countResponse); + assertEquals(12, countResponse.getCount()); + } + + public void testCountOneIndexMatchQuery() throws IOException { + CountRequest countRequest = new CountRequest("index"); + countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10))); + CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync); + assertCountHeader(countResponse); + assertEquals(1, countResponse.getCount()); + } + + public void testCountMultipleIndicesMatchQueryUsingConstructor() throws IOException { + + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder().query(new MatchQueryBuilder("field", "value1")); + CountRequest countRequest = new CountRequest(new String[]{"index1", "index2", "index3"}, sourceBuilder); + CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync); + assertCountHeader(countResponse); + assertEquals(3, countResponse.getCount()); + + } + + public void testCountMultipleIndicesMatchQuery() throws IOException { + + CountRequest countRequest = new CountRequest("index1", "index2", "index3"); + countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("field", "value1"))); + CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync); + assertCountHeader(countResponse); + assertEquals(3, countResponse.getCount()); + } + + public void testCountAllIndicesMatchQuery() throws IOException { + + CountRequest countRequest = new CountRequest(); + countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("field", "value1"))); + CountResponse countResponse = execute(countRequest, highLevelClient()::count, highLevelClient()::countAsync); + assertCountHeader(countResponse); + assertEquals(3, countResponse.getCount()); + } + + private static void assertCountHeader(CountResponse countResponse) { + assertEquals(0, countResponse.getSkippedShards()); + assertEquals(0, countResponse.getFailedShards()); + assertThat(countResponse.getTotalShards(), greaterThan(0)); + assertEquals(countResponse.getTotalShards(), countResponse.getSuccessfulShards()); + assertEquals(0, countResponse.getShardFailures().length); + } } diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java new file mode 100644 index 00000000000..1030f4401e1 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountRequestTests.java @@ -0,0 +1,95 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.core; + +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.util.ArrayUtils; +import org.elasticsearch.index.query.MatchQueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.test.ESTestCase; + +import java.util.ArrayList; +import java.util.List; + +import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode; + +//similar to SearchRequestTests as CountRequest inline several members (and functionality) from SearchRequest +public class CountRequestTests extends ESTestCase { + + public void testIllegalArguments() { + CountRequest countRequest = new CountRequest(); + assertNotNull(countRequest.indices()); + assertNotNull(countRequest.indicesOptions()); + assertNotNull(countRequest.types()); + + NullPointerException e = expectThrows(NullPointerException.class, () -> countRequest.indices((String[]) null)); + assertEquals("indices must not be null", e.getMessage()); + e = expectThrows(NullPointerException.class, () -> countRequest.indices((String) null)); + assertEquals("index must not be null", e.getMessage()); + + e = expectThrows(NullPointerException.class, () -> countRequest.indicesOptions(null)); + assertEquals("indicesOptions must not be null", e.getMessage()); + + e = expectThrows(NullPointerException.class, () -> countRequest.types((String[]) null)); + assertEquals("types must not be null", e.getMessage()); + e = expectThrows(NullPointerException.class, () -> countRequest.types((String) null)); + assertEquals("type must not be null", e.getMessage()); + + e = expectThrows(NullPointerException.class, () -> countRequest.source(null)); + assertEquals("source must not be null", e.getMessage()); + + } + + public void testEqualsAndHashcode() { + checkEqualsAndHashCode(createCountRequest(), CountRequestTests::copyRequest, this::mutate); + } + + private CountRequest createCountRequest() { + CountRequest countRequest = new CountRequest("index"); + countRequest.source(new SearchSourceBuilder().query(new MatchQueryBuilder("num", 10))); + return countRequest; + } + + private CountRequest mutate(CountRequest countRequest) { + CountRequest mutation = copyRequest(countRequest); + List mutators = new ArrayList<>(); + mutators.add(() -> mutation.indices(ArrayUtils.concat(countRequest.indices(), new String[]{randomAlphaOfLength(10)}))); + mutators.add(() -> mutation.indicesOptions(randomValueOtherThan(countRequest.indicesOptions(), + () -> IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean())))); + mutators.add(() -> mutation.types(ArrayUtils.concat(countRequest.types(), new String[]{randomAlphaOfLength(10)}))); + mutators.add(() -> mutation.preference(randomValueOtherThan(countRequest.preference(), () -> randomAlphaOfLengthBetween(3, 10)))); + mutators.add(() -> mutation.routing(randomValueOtherThan(countRequest.routing(), () -> randomAlphaOfLengthBetween(3, 10)))); + randomFrom(mutators).run(); + return mutation; + } + + private static CountRequest copyRequest(CountRequest countRequest) { + CountRequest result = new CountRequest(); + result.indices(countRequest.indices()); + result.indicesOptions(countRequest.indicesOptions()); + result.types(countRequest.types()); + result.routing(countRequest.routing()); + result.preference(countRequest.preference()); + if (countRequest.source() != null) { + result.source(countRequest.source()); + } + return result; + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java new file mode 100644 index 00000000000..c2fc668d604 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/core/CountResponseTests.java @@ -0,0 +1,126 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.core; + +import org.elasticsearch.action.search.ShardSearchFailure; +import org.elasticsearch.cluster.metadata.IndexMetaData; +import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.Index; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.rest.action.RestActions; +import org.elasticsearch.search.SearchShardTarget; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.elasticsearch.test.AbstractXContentTestCase.xContentTester; + +public class CountResponseTests extends ESTestCase { + + // Not comparing XContent for equivalence as we cannot compare the ShardSearchFailure#cause, because it will be wrapped in an outer + // ElasticSearchException. Best effort: try to check that the original message appears somewhere in the rendered xContent + // For more see ShardSearchFailureTests. + public void testFromXContent() throws IOException { + xContentTester( + this::createParser, + this::createTestInstance, + this::toXContent, + CountResponse::fromXContent) + .supportsUnknownFields(false) + .assertEqualsConsumer(this::assertEqualInstances) + .assertToXContentEquivalence(false) + .test(); + } + + private CountResponse createTestInstance() { + long count = 5; + Boolean terminatedEarly = randomBoolean() ? null : randomBoolean(); + int totalShards = randomIntBetween(1, Integer.MAX_VALUE); + int successfulShards = randomIntBetween(0, totalShards); + int skippedShards = randomIntBetween(0, totalShards); + int numFailures = randomIntBetween(1, 5); + ShardSearchFailure[] failures = new ShardSearchFailure[numFailures]; + for (int i = 0; i < failures.length; i++) { + failures[i] = createShardFailureTestItem(); + } + CountResponse.ShardStats shardStats = new CountResponse.ShardStats(successfulShards, totalShards, skippedShards, + randomBoolean() ? ShardSearchFailure.EMPTY_ARRAY : failures); + return new CountResponse(count, terminatedEarly, shardStats); + } + + private void toXContent(CountResponse response, XContentBuilder builder) throws IOException { + builder.startObject(); + builder.field(CountResponse.COUNT.getPreferredName(), response.getCount()); + if (response.isTerminatedEarly() != null) { + builder.field(CountResponse.TERMINATED_EARLY.getPreferredName(), response.isTerminatedEarly()); + } + toXContent(response.getShardStats(), builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + } + + private void toXContent(CountResponse.ShardStats stats, XContentBuilder builder, ToXContent.Params params) throws IOException { + RestActions.buildBroadcastShardsHeader(builder, params, stats.getTotalShards(), stats.getSuccessfulShards(), stats + .getSkippedShards(), stats.getShardFailures().length, stats.getShardFailures()); + } + + @SuppressWarnings("Duplicates") + private static ShardSearchFailure createShardFailureTestItem() { + String randomMessage = randomAlphaOfLengthBetween(3, 20); + Exception ex = new ParsingException(0, 0, randomMessage, new IllegalArgumentException("some bad argument")); + SearchShardTarget searchShardTarget = null; + if (randomBoolean()) { + String nodeId = randomAlphaOfLengthBetween(5, 10); + String indexName = randomAlphaOfLengthBetween(5, 10); + searchShardTarget = new SearchShardTarget(nodeId, + new ShardId(new Index(indexName, IndexMetaData.INDEX_UUID_NA_VALUE), randomInt()), null, null); + } + return new ShardSearchFailure(ex, searchShardTarget); + } + + private void assertEqualInstances(CountResponse expectedInstance, CountResponse newInstance) { + assertEquals(expectedInstance.getCount(), newInstance.getCount()); + assertEquals(expectedInstance.status(), newInstance.status()); + assertEquals(expectedInstance.isTerminatedEarly(), newInstance.isTerminatedEarly()); + assertEquals(expectedInstance.getTotalShards(), newInstance.getTotalShards()); + assertEquals(expectedInstance.getFailedShards(), newInstance.getFailedShards()); + assertEquals(expectedInstance.getSkippedShards(), newInstance.getSkippedShards()); + assertEquals(expectedInstance.getSuccessfulShards(), newInstance.getSuccessfulShards()); + assertEquals(expectedInstance.getShardFailures().length, newInstance.getShardFailures().length); + + ShardSearchFailure[] expectedFailures = expectedInstance.getShardFailures(); + ShardSearchFailure[] newFailures = newInstance.getShardFailures(); + + for (int i = 0; i < newFailures.length; i++) { + ShardSearchFailure parsedFailure = newFailures[i]; + ShardSearchFailure originalFailure = expectedFailures[i]; + assertEquals(originalFailure.index(), parsedFailure.index()); + assertEquals(originalFailure.shard(), parsedFailure.shard()); + assertEquals(originalFailure.shardId(), parsedFailure.shardId()); + String originalMsg = originalFailure.getCause().getMessage(); + assertEquals(parsedFailure.getCause().getMessage(), "Elasticsearch exception [type=parsing_exception, reason=" + + originalMsg + "]"); + String nestedMsg = originalFailure.getCause().getCause().getMessage(); + assertEquals(parsedFailure.getCause().getCause().getMessage(), + "Elasticsearch exception [type=illegal_argument_exception, reason=" + nestedMsg + "]"); + } + } +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java index 1e596158750..831c39ed28b 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/SearchDocumentationIT.java @@ -49,6 +49,8 @@ import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.Response; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestHighLevelClient; +import org.elasticsearch.client.core.CountRequest; +import org.elasticsearch.client.core.CountResponse; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.text.Text; @@ -1287,4 +1289,124 @@ public class SearchDocumentationIT extends ESRestHighLevelClientTestCase { assertSame(RestStatus.OK, bulkResponse.status()); assertFalse(bulkResponse.hasFailures()); } + + + @SuppressWarnings({"unused", "unchecked"}) + public void testCount() throws Exception { + indexCountTestData(); + RestHighLevelClient client = highLevelClient(); + { + // tag::count-request-basic + CountRequest countRequest = new CountRequest(); // <1> + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); // <2> + searchSourceBuilder.query(QueryBuilders.matchAllQuery()); // <3> + countRequest.source(searchSourceBuilder); // <4> + // end::count-request-basic + } + { + // tag::count-request-indices-types + CountRequest countRequest = new CountRequest("blog"); // <1> + countRequest.types("doc"); // <2> + // end::count-request-indices-types + // tag::count-request-routing + countRequest.routing("routing"); // <1> + // end::count-request-routing + // tag::count-request-indicesOptions + countRequest.indicesOptions(IndicesOptions.lenientExpandOpen()); // <1> + // end::count-request-indicesOptions + // tag::count-request-preference + countRequest.preference("_local"); // <1> + // end::count-request-preference + assertNotNull(client.count(countRequest, RequestOptions.DEFAULT)); + } + { + // tag::count-source-basics + SearchSourceBuilder sourceBuilder = new SearchSourceBuilder(); // <1> + sourceBuilder.query(QueryBuilders.termQuery("user", "kimchy")); // <2> + // end::count-source-basics + + // tag::count-source-setter + CountRequest countRequest = new CountRequest(); + countRequest.indices("blog", "author"); + countRequest.source(sourceBuilder); + // end::count-source-setter + + // tag::count-execute + CountResponse countResponse = client + .count(countRequest, RequestOptions.DEFAULT); + // end::count-execute + + // tag::count-execute-listener + ActionListener listener = + new ActionListener() { + + @Override + public void onResponse(CountResponse countResponse) { + // <1> + } + + @Override + public void onFailure(Exception e) { + // <2> + } + }; + // end::count-execute-listener + + // Replace the empty listener by a blocking listener in test + final CountDownLatch latch = new CountDownLatch(1); + listener = new LatchedActionListener<>(listener, latch); + + // tag::count-execute-async + client.countAsync(countRequest, RequestOptions.DEFAULT, listener); // <1> + // end::count-execute-async + + assertTrue(latch.await(30L, TimeUnit.SECONDS)); + + // tag::count-response-1 + long count = countResponse.getCount(); + RestStatus status = countResponse.status(); + Boolean terminatedEarly = countResponse.isTerminatedEarly(); + // end::count-response-1 + + // tag::count-response-2 + int totalShards = countResponse.getTotalShards(); + int skippedShards = countResponse.getSkippedShards(); + int successfulShards = countResponse.getSuccessfulShards(); + int failedShards = countResponse.getFailedShards(); + for (ShardSearchFailure failure : countResponse.getShardFailures()) { + // failures should be handled here + } + // end::count-response-2 + assertNotNull(countResponse); + assertEquals(4, countResponse.getCount()); + } + } + + private static void indexCountTestData() throws IOException { + CreateIndexRequest authorsRequest = new CreateIndexRequest("author") + .mapping("doc", "user", "type=keyword,doc_values=false"); + CreateIndexResponse authorsResponse = highLevelClient().indices().create(authorsRequest, RequestOptions.DEFAULT); + assertTrue(authorsResponse.isAcknowledged()); + + BulkRequest bulkRequest = new BulkRequest(); + bulkRequest.add(new IndexRequest("blog", "doc", "1") + .source(XContentType.JSON, "title", "Doubling Down on Open?", "user", + Collections.singletonList("kimchy"), "innerObject", Collections.singletonMap("key", "value"))); + bulkRequest.add(new IndexRequest("blog", "doc", "2") + .source(XContentType.JSON, "title", "Swiftype Joins Forces with Elastic", "user", + Arrays.asList("kimchy", "matt"), "innerObject", Collections.singletonMap("key", "value"))); + bulkRequest.add(new IndexRequest("blog", "doc", "3") + .source(XContentType.JSON, "title", "On Net Neutrality", "user", + Arrays.asList("tyler", "kimchy"), "innerObject", Collections.singletonMap("key", "value"))); + + bulkRequest.add(new IndexRequest("author", "doc", "1") + .source(XContentType.JSON, "user", "kimchy")); + + + bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = highLevelClient().bulk(bulkRequest, RequestOptions.DEFAULT); + assertSame(RestStatus.OK, bulkResponse.status()); + assertFalse(bulkResponse.hasFailures()); + } + } diff --git a/docs/java-rest/high-level/search/count.asciidoc b/docs/java-rest/high-level/search/count.asciidoc new file mode 100644 index 00000000000..f70e1e1fd4d --- /dev/null +++ b/docs/java-rest/high-level/search/count.asciidoc @@ -0,0 +1,114 @@ +-- +:api: count +:request: CountRequest +:response: CountResponse +-- +[id="{upid}-{api}"] + +=== Count API + +[id="{upid}-{api}-request"] + +==== Count Request + +The +{request}+ is used to execute a query and get the number of matches for the query. The query to use in +{request}+ can be +set in similar way as query in `SearchRequest` using `SearchSourceBuilder`. + +In its most basic form, we can add a query to the request: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-basic] +-------------------------------------------------- + +<1> Creates the +{request}+. Without arguments this runs against all indices. +<2> Most search parameters are added to the `SearchSourceBuilder`. +<3> Add a `match_all` query to the `SearchSourceBuilder`. +<4> Add the `SearchSourceBuilder` to the +{request}+. + +[[java-rest-high-count-request-optional]] +===== Count Request optional arguments + +Let's first look at some of the optional arguments of a +{request}+: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-indices-types] +-------------------------------------------------- +<1> Restricts the request to an index +<2> Limits the request to a type + +There are a couple of other interesting optional parameters: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-routing] +-------------------------------------------------- +<1> Set a routing parameter + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-indicesOptions] +-------------------------------------------------- +<1> Setting `IndicesOptions` controls how unavailable indices are resolved and how wildcard expressions are expanded + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-request-preference] +-------------------------------------------------- +<1> Use the preference parameter e.g. to execute the search to prefer local shards. The default is to randomize across shards. + +===== Using the SearchSourceBuilder in CountRequest + +Both in search and count API calls, most options controlling the search behavior can be set on the `SearchSourceBuilder`, +which contains more or less the equivalent of the options in the search request body of the Rest API. + +Here are a few examples of some common options: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-source-basics] +-------------------------------------------------- +<1> Create a `SearchSourceBuilder` with default options. +<2> Set the query. Can be any type of `QueryBuilder` + +After this, the `SearchSourceBuilder` only needs to be added to the ++{request}+: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-source-setter] +-------------------------------------------------- + +Note subtle difference when using `SearchSourceBuilder` in `SearchRequest` and using `SearchSourceBuilder` in +{request}+ - using +`SearchSourceBuilder` in `SearchRequest` one can use `SearchSourceBuilder.size()` and `SearchSourceBuilder.from()` methods to set the +number of search hits to return, and the starting index. In +{request}+ we're interested in total number of matches and these methods +have no meaning. + +The <> page gives a list of all available search queries with +their corresponding `QueryBuilder` objects and `QueryBuilders` helper methods. + +include::../execution.asciidoc[] + +[id="{upid}-{api}-response"] +==== CountResponse + +The +{response}+ that is returned by executing the count API call provides total count of hits and details about the count execution +itself, like the HTTP status code, or whether the request terminated early: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response-1] +-------------------------------------------------- + +The response also provides information about the execution on the +shard level by offering statistics about the total number of shards that were +affected by the underlying search, and the successful vs. unsuccessful shards. Possible +failures can also be handled by iterating over an array off +`ShardSearchFailures` like in the following example: + +["source","java",subs="attributes,callouts,macros"] +-------------------------------------------------- +include-tagged::{doc-tests-file}[{api}-response-2] +-------------------------------------------------- + diff --git a/docs/java-rest/high-level/supported-apis.asciidoc b/docs/java-rest/high-level/supported-apis.asciidoc index 4411a6b375f..dc17296e5c2 100644 --- a/docs/java-rest/high-level/supported-apis.asciidoc +++ b/docs/java-rest/high-level/supported-apis.asciidoc @@ -54,6 +54,7 @@ The Java High Level REST Client supports the following Search APIs: * <<{upid}-field-caps>> * <<{upid}-rank-eval>> * <<{upid}-explain>> +* <<{upid}-count>> include::search/search.asciidoc[] include::search/scroll.asciidoc[] @@ -63,6 +64,7 @@ include::search/multi-search-template.asciidoc[] include::search/field-caps.asciidoc[] include::search/rank-eval.asciidoc[] include::search/explain.asciidoc[] +include::search/count.asciidoc[] == Miscellaneous APIs