HLRC: Add rollup search (#36334)

Relates to #29827

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -354,8 +354,15 @@ final class RequestConverters {
         return request;
     }
 
-    static Request search(SearchRequest searchRequest) throws IOException {
-        Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchRequest.types(), "_search"));
+    /**
+     * Convert a {@linkplain SearchRequest} into a {@linkplain Request}.
+     * @param searchRequest the request to convert
+     * @param searchEndpoint the name of the search endpoint. {@literal _search}
+     *    for standard searches and {@literal _rollup_search} for rollup
+     *    searches.
+     */
+    static Request search(SearchRequest searchRequest, String searchEndpoint) throws IOException {
+        Request request = new Request(HttpPost.METHOD_NAME, endpoint(searchRequest.indices(), searchRequest.types(), searchEndpoint));
 
         Params params = new Params(request);
         addSearchRequestParams(params, searchRequest);

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -908,7 +908,12 @@ public class RestHighLevelClient implements Closeable {
      * @return the response
      */
     public final SearchResponse search(SearchRequest searchRequest, RequestOptions options) throws IOException {
-        return performRequestAndParseEntity(searchRequest, RequestConverters::search, options, SearchResponse::fromXContent, emptySet());
+        return performRequestAndParseEntity(
+                searchRequest,
+                r -> RequestConverters.search(r, "_search"),
+                options,
+                SearchResponse::fromXContent,
+                emptySet());
     }
 
     /**
@@ -919,7 +924,12 @@ public class RestHighLevelClient implements Closeable {
      * @param listener the listener to be notified upon request completion
      */
     public final void searchAsync(SearchRequest searchRequest, RequestOptions options, ActionListener<SearchResponse> listener) {
-        performRequestAsyncAndParseEntity(searchRequest, RequestConverters::search, options, SearchResponse::fromXContent, listener,
+        performRequestAsyncAndParseEntity(
+                searchRequest,
+                r -> RequestConverters.search(r, "_search"),
+                options,
+                SearchResponse::fromXContent,
+                listener,
                 emptySet());
     }
 

client/rest-high-level/src/main/java/org/elasticsearch/client/RollupClient.java

@@ -20,6 +20,8 @@
 package org.elasticsearch.client;
 
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.core.AcknowledgedResponse;
 import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupIndexCapsRequest;
@@ -224,6 +226,42 @@ public class RollupClient {
             listener, Collections.emptySet());
     }
 
+    /**
+     * Perform a rollup search.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-search.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public SearchResponse search(SearchRequest request, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(
+                request,
+                RollupRequestConverters::search,
+                options,
+                SearchResponse::fromXContent,
+                Collections.emptySet());
+    }
+
+    /**
+     * Perform a rollup search.
+     * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/rollup-search.html">
+     * the docs</a> for more.
+     * @param request the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void searchAsync(SearchRequest request, RequestOptions options, ActionListener<SearchResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(
+                request,
+                RollupRequestConverters::search,
+                options,
+                SearchResponse::fromXContent,
+                listener,
+                Collections.emptySet());
+    }
+
     /**
      * Get the Rollup Capabilities of a target (non-rollup) index or pattern
      * See <a href="https://www.elastic.co/guide/en/elasticsearch/reference/master/rollup-get-rollup-caps.html">

client/rest-high-level/src/main/java/org/elasticsearch/client/RollupRequestConverters.java

@@ -22,6 +22,7 @@ import org.apache.http.client.methods.HttpDelete;
 import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.methods.HttpPut;
+import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.client.rollup.DeleteRollupJobRequest;
 import org.elasticsearch.client.rollup.GetRollupCapsRequest;
 import org.elasticsearch.client.rollup.GetRollupIndexCapsRequest;
@@ -93,6 +94,20 @@ final class RollupRequestConverters {
         return request;
     }
 
+    static Request search(final SearchRequest request) throws IOException {
+        if (request.types().length > 0) {
+            /*
+             * Ideally we'd check this with the standard validation framework
+             * but we don't have a special request for rollup search so that'd
+             * be difficult. 
+             */
+            ValidationException ve = new ValidationException();
+            ve.addValidationError("types are not allowed in rollup search");
+            throw ve;
+        }
+        return RequestConverters.search(request, "_rollup_search");
+    }
+
     static Request getRollupCaps(final GetRollupCapsRequest getRollupCapsRequest) throws IOException {
         String endpoint = new RequestConverters.EndpointBuilder()
             .addPathPartAsIs("_xpack", "rollup", "data")

client/rest-high-level/src/test/java/org/elasticsearch/client/RequestConvertersTests.java

@@ -880,14 +880,16 @@ public class RequestConvertersTests extends ESTestCase {
     }
 
     public void testSearchNullSource() throws IOException {
+        String searchEndpoint = randomFrom("_" + randomAlphaOfLength(5));
         SearchRequest searchRequest = new SearchRequest();
-        Request request = RequestConverters.search(searchRequest);
+        Request request = RequestConverters.search(searchRequest, searchEndpoint);
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
-        assertEquals("/_search", request.getEndpoint());
+        assertEquals("/" + searchEndpoint, request.getEndpoint());
         assertNull(request.getEntity());
     }
 
     public void testSearch() throws Exception {
+        String searchEndpoint = randomFrom("_" + randomAlphaOfLength(5));
         String[] indices = randomIndicesNames(0, 5);
         SearchRequest searchRequest = new SearchRequest(indices);
 
@@ -948,7 +950,7 @@ public class RequestConvertersTests extends ESTestCase {
             searchRequest.source(searchSourceBuilder);
         }
 
-        Request request = RequestConverters.search(searchRequest);
+        Request request = RequestConverters.search(searchRequest, searchEndpoint);
         StringJoiner endpoint = new StringJoiner("/", "/", "");
         String index = String.join(",", indices);
         if (Strings.hasLength(index)) {
@@ -958,7 +960,7 @@ public class RequestConvertersTests extends ESTestCase {
         if (Strings.hasLength(type)) {
             endpoint.add(type);
         }
-        endpoint.add("_search");
+        endpoint.add(searchEndpoint);
         assertEquals(HttpPost.METHOD_NAME, request.getMethod());
         assertEquals(endpoint.toString(), request.getEndpoint());
         assertEquals(expectedParams, request.getParameters());

client/rest-high-level/src/test/java/org/elasticsearch/client/RollupIT.java

@@ -54,11 +54,13 @@ import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.SearchHit;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
 import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.Before;
 
 import java.util.Arrays;
@@ -70,6 +72,7 @@ import java.util.Map;
 import java.util.Set;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.either;
 import static org.hamcrest.Matchers.empty;
 import static org.hamcrest.Matchers.equalTo;
@@ -244,6 +247,33 @@ public class RollupIT extends ESRestHighLevelClientTestCase {
         }
     }
 
+    public void testSearch() throws Exception {
+        testPutStartAndGetRollupJob();
+        SearchRequest search = new SearchRequest(rollupIndex);
+        search.source(new SearchSourceBuilder()
+                .size(0)
+                .aggregation(new AvgAggregationBuilder("avg").field("value")));
+        SearchResponse response = highLevelClient().rollup().search(search, RequestOptions.DEFAULT);
+        assertEquals(0, response.getFailedShards());
+        assertEquals(0, response.getHits().getTotalHits().value);
+        NumericMetricsAggregation.SingleValue avg = response.getAggregations().get("avg");
+        assertThat(avg.value(), closeTo(sum / numDocs, 0.00000001));
+    }
+
+    public void testSearchWithType() throws Exception {
+        SearchRequest search = new SearchRequest(rollupIndex);
+        search.types("a", "b", "c");
+        search.source(new SearchSourceBuilder()
+                .size(0)
+                .aggregation(new AvgAggregationBuilder("avg").field("value")));
+        try {
+            highLevelClient().rollup().search(search, RequestOptions.DEFAULT);
+            fail("types are not allowed but didn't fail");
+        } catch (ValidationException e) {
+            assertEquals("Validation Failed: 1: types are not allowed in rollup search;", e.getMessage());
+        }
+    }
+
     public void testGetMissingRollupJob() throws Exception {
         GetRollupJobRequest getRollupJobRequest = new GetRollupJobRequest("missing");
         RollupClient rollupClient = highLevelClient().rollup();

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/RollupDocumentationIT.java

@@ -27,6 +27,8 @@ import org.elasticsearch.action.admin.indices.refresh.RefreshResponse;
 import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.bulk.BulkResponse;
 import org.elasticsearch.action.index.IndexRequest;
+import org.elasticsearch.action.search.SearchRequest;
+import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.action.support.WriteRequest;
 import org.elasticsearch.client.ESRestHighLevelClientTestCase;
 import org.elasticsearch.client.RequestOptions;
@@ -60,6 +62,9 @@ import org.elasticsearch.cluster.health.ClusterHealthStatus;
 import org.elasticsearch.common.unit.TimeValue;
 import org.elasticsearch.rest.RestStatus;
 import org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval;
+import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
+import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
 import org.junit.Before;
 
 import java.io.IOException;
@@ -72,6 +77,7 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder;
+import static org.hamcrest.Matchers.closeTo;
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.hasSize;
 import static org.hamcrest.Matchers.isOneOf;
@@ -89,7 +95,7 @@ public class RollupDocumentationIT extends ESRestHighLevelClientTestCase {
                 .field("timestamp", String.format(Locale.ROOT, "2018-01-01T00:%02d:00Z", i))
                 .field("hostname", 0)
                 .field("datacenter", 0)
-                .field("temperature", 0)
+                .field("temperature", i)
                 .field("voltage", 0)
                 .field("load", 0)
                 .field("net_in", 0)
@@ -330,6 +336,56 @@ public class RollupDocumentationIT extends ESRestHighLevelClientTestCase {
         assertTrue(latch.await(30L, TimeUnit.SECONDS));
     }
 
+    public void testSearch() throws Exception {
+        // Setup a rollup index to query
+        testCreateRollupJob();
+
+        RestHighLevelClient client = highLevelClient();
+
+        // tag::search-request
+        SearchRequest request = new SearchRequest();
+        request.source(new SearchSourceBuilder()
+            .size(0)
+            .aggregation(new MaxAggregationBuilder("max_temperature")
+                .field("temperature")));
+        // end::search-request
+
+        // tag::search-execute
+        SearchResponse response =
+            client.rollup().search(request, RequestOptions.DEFAULT);
+        // end::search-execute
+
+        // tag::search-response
+        NumericMetricsAggregation.SingleValue maxTemperature =
+                response.getAggregations().get("max_temperature");
+        assertThat(maxTemperature.value(), closeTo(49.0, .00001));
+        // end::search-response
+
+        ActionListener<SearchResponse> listener;
+        // tag::search-execute-listener
+        listener = new ActionListener<SearchResponse>() {
+            @Override
+            public void onResponse(SearchResponse response) {
+                 // <1>
+            }
+
+            @Override
+            public void onFailure(Exception e) {
+                // <2>
+            }
+        };
+        // end::search-execute-listener
+
+        final CountDownLatch latch = new CountDownLatch(1);
+        listener = new LatchedActionListener<>(listener, latch);
+
+        // tag::search-execute-async
+        client.rollup().searchAsync(request, RequestOptions.DEFAULT, listener); // <1>
+        // end::search-execute-async
+
+        assertTrue(latch.await(30L, TimeUnit.SECONDS));
+    }
+
     @SuppressWarnings("unused")
     public void testGetRollupCaps() throws Exception {
         RestHighLevelClient client = highLevelClient();

docs/java-rest/high-level/rollup/search.asciidoc

@@ -0,0 +1,45 @@
+--
+:api: search
+:request: SearchRequest
+:response: SearchResponse
+--
+
+[id="{upid}-{api}"]
+=== Rollup Search API
+
+The Rollup Search endpoint allows searching rolled-up data using the standard
+query DSL. The Rollup Search endpoint is needed because, internally,
+rolled-up documents utilize a different document structure than the original
+data. The Rollup Search endpoint rewrites standard query DSL into a format that
+matches the rollup documents, then takes the response and rewrites it back to
+what a client would expect given the original query.
+
+[id="{upid}-{api}-request"]
+==== Request
+
+Rollup Search uses the same +{request}+ that is used by the <<{mainid}-search>>
+but it is mostly for aggregations you should set the `size` to 0 and add
+aggregations like this:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-request]
+--------------------------------------------------
+
+NOTE:: Rollup Search is limited in many ways because only some query elements
+can be translated into queries against the rollup indices. See the main
+{ref}/rollup-search.html[Rollup Search] documentation for more.
+
+include::../execution.asciidoc[]
+
+[id="{upid}-{api}-response"]
+==== Response
+
+Rollup Search returns the same +{response}+ that is used by the
+<<{mainid}-search>> and everything can be accessed in exactly the same way.
+This will access the aggregation built by the example request above:
+
+["source","java",subs="attributes,callouts,macros"]
+--------------------------------------------------
+include-tagged::{doc-tests-file}[{api}-response]
+--------------------------------------------------

docs/java-rest/high-level/supported-apis.asciidoc

@@ -360,6 +360,7 @@ The Java High Level REST Client supports the following Rollup APIs:
 * <<{upid}-rollup-stop-job>>
 * <<{upid}-rollup-delete-job>>
 * <<java-rest-high-x-pack-rollup-get-job>>
+* <<{upid}-search>>
 * <<{upid}-x-pack-rollup-get-rollup-caps>>
 * <<{upid}-x-pack-rollup-get-rollup-index-caps>>
 
@@ -368,6 +369,7 @@ include::rollup/start_job.asciidoc[]
 include::rollup/stop_job.asciidoc[]
 include::rollup/delete_job.asciidoc[]
 include::rollup/get_job.asciidoc[]
+include::rollup/search.asciidoc[]
 include::rollup/get_rollup_caps.asciidoc[]
 include::rollup/get_rollup_index_caps.asciidoc[]
 

docs/reference/rollup/apis/rollup-search.asciidoc

@@ -26,6 +26,7 @@ and rewrites it back to what a client would expect given the original query.
   indices.
 
 Rules for the `index` parameter:
+
 - At least one index/index-pattern must be specified.  This can be either a rollup or non-rollup index.  Omitting the index parameter,
 or using `_all`, is not permitted
 - Multiple non-rollup indices may be specified

x-pack/plugin/rollup/src/main/java/org/elasticsearch/xpack/rollup/rest/RestRollupSearchAction.java

@@ -16,12 +16,16 @@ import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.xpack.core.rollup.action.RollupSearchAction;
 
 import java.io.IOException;
+import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.Set;
 
 public class RestRollupSearchAction extends BaseRestHandler {
 
-    private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TOTAL_HIT_AS_INT_PARAM);
+    private static final Set<String> RESPONSE_PARAMS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(
+            RestSearchAction.TYPED_KEYS_PARAM,
+            RestSearchAction.TOTAL_HIT_AS_INT_PARAM)));
 
     public RestRollupSearchAction(Settings settings, RestController controller) {
         super(settings);

x-pack/plugin/src/test/resources/rest-api-spec/api/xpack.rollup.rollup_search.json

@@ -18,6 +18,10 @@
         }
       },
       "params": {
+        "typed_keys": {
+          "type" : "boolean",
+          "description" : "Specify whether aggregation and suggester names should be prefixed by their respective types in the response"
+        },
         "rest_total_hits_as_int" : {
           "type" : "boolean",
           "description" : "Indicates whether hits.total should be rendered as an integer or an object in the rest search response",

x-pack/plugin/src/test/resources/rest-api-spec/test/rollup/rollup_search.yml

@@ -904,3 +904,36 @@ setup:
                  interval: "1h"
                  time_zone: "UTC"
 
+
+---
+"Search with typed_keys":
+
+  - do:
+      xpack.rollup.rollup_search:
+        index: "foo_rollup"
+        typed_keys: true
+        body:
+          size: 0
+          aggs:
+            histo:
+              date_histogram:
+                field: "timestamp"
+                interval: "1h"
+                time_zone: "UTC"
+              aggs:
+                the_max:
+                  max:
+                    field: "price"
+
+  - match: { aggregations.date_histogram#histo.buckets.0.key_as_string: "2017-01-01T05:00:00.000Z" }
+  - match: { aggregations.date_histogram#histo.buckets.0.doc_count: 1 }
+  - match: { aggregations.date_histogram#histo.buckets.0.max#the_max.value: 1 }
+  - match: { aggregations.date_histogram#histo.buckets.1.key_as_string: "2017-01-01T06:00:00.000Z" }
+  - match: { aggregations.date_histogram#histo.buckets.1.doc_count: 2 }
+  - match: { aggregations.date_histogram#histo.buckets.1.max#the_max.value: 2 }
+  - match: { aggregations.date_histogram#histo.buckets.2.key_as_string: "2017-01-01T07:00:00.000Z" }
+  - match: { aggregations.date_histogram#histo.buckets.2.doc_count: 10 }
+  - match: { aggregations.date_histogram#histo.buckets.2.max#the_max.value: 3 }
+  - match: { aggregations.date_histogram#histo.buckets.3.key_as_string: "2017-01-01T08:00:00.000Z" }
+  - match: { aggregations.date_histogram#histo.buckets.3.doc_count: 20 }
+  - match: { aggregations.date_histogram#histo.buckets.3.max#the_max.value: 4 }