Add parameter to prefix aggs name with type in search responses (#22965)

This pull request adds a new parameter to the REST Search API named `typed_keys`. When set to true, the aggregation names in the search response will be prefixed with a prefix that reflects the internal type of the aggregation.

Here is a simple example:
```
GET /_search?typed_keys
{
    "aggs": {
        "tweets_per_user": {
            "terms": {
                "field": "user"
            }
        }
    },
    "size": 0
}
```

And the response:

```
{
    "aggs": {
        "sterms:tweets_per_user": {
            ...
        }
    }
}
```

This parameter is intended to make life easier for REST clients that could parse back the prefix and could detect the type of the aggregation to parse. It could also be implemented for suggesters.
This commit is contained in:
Tanguy Leroux 2017-02-09 11:19:04 +01:00 committed by GitHub
parent e02d5563f4
commit 3553522328
17 changed files with 599 additions and 1 deletions

View File

@ -39,7 +39,9 @@ import org.elasticsearch.rest.action.RestToXContentListener;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.function.BiConsumer;
import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
@ -50,6 +52,8 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
public class RestMultiSearchAction extends BaseRestHandler {
private static final Set<String> RESPONSE_PARAMS = Collections.singleton("typed_keys");
private final boolean allowExplicitIndex;
public RestMultiSearchAction(Settings settings, RestController controller) {
@ -199,4 +203,9 @@ public class RestMultiSearchAction extends BaseRestHandler {
}
return -1;
}
@Override
protected Set<String> responseParams() {
return RESPONSE_PARAMS;
}
}

View File

@ -44,6 +44,8 @@ import org.elasticsearch.search.suggest.term.TermSuggestionBuilder.SuggestMode;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.Set;
import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
import static org.elasticsearch.rest.RestRequest.Method.GET;
@ -51,6 +53,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
import static org.elasticsearch.search.suggest.SuggestBuilders.termSuggestion;
public class RestSearchAction extends BaseRestHandler {
private static final Set<String> RESPONSE_PARAMS = Collections.singleton("typed_keys");
public RestSearchAction(Settings settings, RestController controller) {
super(settings);
controller.registerHandler(GET, "/_search", this);
@ -219,4 +224,9 @@ public class RestSearchAction extends BaseRestHandler {
.suggestMode(SuggestMode.resolve(suggestMode))));
}
}
@Override
protected Set<String> responseParams() {
return RESPONSE_PARAMS;
}
}

View File

@ -38,6 +38,10 @@ import java.util.Objects;
* An internal implementation of {@link Aggregation}. Serves as a base class for all aggregation implementations.
*/
public abstract class InternalAggregation implements Aggregation, ToXContent, NamedWriteable {
/** Delimiter used when prefixing aggregation names with their type using the typed_keys parameter **/
public static final String TYPED_KEYS_DELIMITER = "#";
public static class ReduceContext {
private final BigArrays bigArrays;
@ -149,8 +153,19 @@ public abstract class InternalAggregation implements Aggregation, ToXContent, Na
return pipelineAggregators;
}
/**
* Returns a string representing the type of the aggregation. This type is added to
* the aggregation name in the response, so that it can later be used by REST clients
* to determine the internal type of the aggregation.
*/
protected String getType() {
return getWriteableName();
}
@Override
public final XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
// Concatenates the type and the name of the aggregation (ex: top_hits#foo)
String name = params.paramAsBoolean("typed_keys", false) ? String.join(TYPED_KEYS_DELIMITER, getType(), getName()) : getName();
builder.startObject(name);
if (this.metaData != null) {
builder.field(CommonFields.META);

View File

@ -45,6 +45,11 @@ public class InternalSampler extends InternalSingleBucketAggregation implements
return NAME;
}
@Override
protected String getType() {
return "sampler";
}
@Override
protected InternalSingleBucketAggregation newAggregation(String name, long docCount,
InternalAggregations subAggregations) {

View File

@ -75,6 +75,11 @@ public class UnmappedSignificantTerms extends InternalSignificantTerms<UnmappedS
return NAME;
}
@Override
protected String getType() {
return SignificantStringTerms.NAME;
}
@Override
public UnmappedSignificantTerms create(List<Bucket> buckets) {
return new UnmappedSignificantTerms(name, requiredSize, minDocCount, pipelineAggregators(), metaData);

View File

@ -71,6 +71,11 @@ public class UnmappedTerms extends InternalTerms<UnmappedTerms, UnmappedTerms.Bu
return NAME;
}
@Override
protected String getType() {
return StringTerms.NAME;
}
@Override
public UnmappedTerms create(List<Bucket> buckets) {
return new UnmappedTerms(name, order, requiredSize, minDocCount, pipelineAggregators(), metaData);

View File

@ -23,6 +23,7 @@ import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentType;
@ -268,7 +269,7 @@ public class SignificantTermsSignificanceScoreIT extends ESIntegTestCase {
XContentBuilder responseBuilder = XContentFactory.jsonBuilder();
responseBuilder.startObject();
classes.toXContent(responseBuilder, null);
classes.toXContent(responseBuilder, ToXContent.EMPTY_PARAMS);
responseBuilder.endObject();
String result = "{\"class\":{\"doc_count_error_upper_bound\":0,\"sum_other_doc_count\":0,"

View File

@ -82,3 +82,89 @@ Then that piece of metadata will be returned in place for our `titles` terms agg
}
--------------------------------------------------
// TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/]
[[returning-aggregation-type]]
== Returning the type of the aggregation
Sometimes you need to know the exact type of an aggregation in order to parse its results. The `typed_keys` parameter
can be used to change the aggregation's name in the response so that it will be prefixed by its internal type.
Considering the following <<search-aggregations-bucket-datehistogram-aggregation,`date_histogram` aggregation>> named
`tweets_over_time` which has a sub <<search-aggregations-metrics-top-hits-aggregation, 'top_hits` aggregation>> named
`top_users`:
[source,js]
--------------------------------------------------
GET /twitter/tweet/_search?typed_keys
{
"aggregations": {
"tweets_over_time": {
"date_histogram": {
"field": "date",
"interval": "year"
},
"aggregations": {
"top_users": {
"top_hits": {
"size": 1
}
}
}
}
}
}
--------------------------------------------------
// CONSOLE
// TEST[setup:twitter]
In the response, the aggregations names will be changed to respectively `date_histogram:tweets_over_time` and
`top_hits:top_users`, reflecting the internal types of each aggregation:
[source,js]
--------------------------------------------------
{
"aggregations": {
"date_histogram#tweets_over_time": { <1>
"buckets" : [
{
"key_as_string" : "2009-01-01T00:00:00.000Z",
"key" : 1230768000000,
"doc_count" : 5,
"top_hits#top_users" : { <2>
"hits" : {
"total" : 5,
"max_score" : 1.0,
"hits" : [
{
"_index": "twitter",
"_type": "tweet",
"_id": "0",
"_score": 1.0,
"_source": {
"date": "2009-11-15T14:12:12",
"message": "trying out Elasticsearch",
"user": "kimchy",
"likes": 0
}
}
]
}
}
}
]
}
},
...
}
--------------------------------------------------
// TESTRESPONSE[s/\.\.\./"took": "$body.took", "timed_out": false, "_shards": "$body._shards", "hits": "$body.hits"/]
<1> The name `tweets_over_time` now contains the `date_histogram` prefix.
<2> The name `top_users` now contains the `top_hits` prefix.
NOTE: For some aggregations, it is possible that the returned type is not the same as the one provided with the
request. This is the case for Terms, Significant Terms and Percentiles aggregations, where the returned type
also contains information about the type of the targeted field: `lterms` (for a terms aggregation on a Long field),
`sigsterms` (for a significant terms aggregation on a String field), `tdigest_percentiles` (for a percentile
aggregation based on the TDigest algorithm).

View File

@ -30,12 +30,16 @@ import org.elasticsearch.rest.action.RestToXContentListener;
import org.elasticsearch.rest.action.search.RestMultiSearchAction;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import static org.elasticsearch.rest.RestRequest.Method.GET;
import static org.elasticsearch.rest.RestRequest.Method.POST;
public class RestMultiSearchTemplateAction extends BaseRestHandler {
private static final Set<String> RESPONSE_PARAMS = Collections.singleton("typed_keys");
private final boolean allowExplicitIndex;
public RestMultiSearchTemplateAction(Settings settings, RestController controller) {
@ -86,4 +90,9 @@ public class RestMultiSearchTemplateAction extends BaseRestHandler {
public boolean supportsContentStream() {
return true;
}
@Override
protected Set<String> responseParams() {
return RESPONSE_PARAMS;
}
}

View File

@ -37,12 +37,16 @@ import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.script.ScriptType;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
import static org.elasticsearch.rest.RestRequest.Method.GET;
import static org.elasticsearch.rest.RestRequest.Method.POST;
public class RestSearchTemplateAction extends BaseRestHandler {
private static final Set<String> RESPONSE_PARAMS = Collections.singleton("typed_keys");
private static final ObjectParser<SearchTemplateRequest, Void> PARSER;
static {
PARSER = new ObjectParser<>("search_template");
@ -107,4 +111,9 @@ public class RestSearchTemplateAction extends BaseRestHandler {
public static SearchTemplateRequest parse(XContentParser parser) throws IOException {
return PARSER.parse(parser, new SearchTemplateRequest(), null);
}
@Override
protected Set<String> responseParams() {
return RESPONSE_PARAMS;
}
}

View File

@ -0,0 +1,119 @@
setup:
- do:
indices.put_template:
name: index_template
body:
index_patterns: test-*
settings:
number_of_replicas: 0
mappings:
user:
properties:
ip:
type: ip
integer:
type: integer
float:
type: float
name:
type: keyword
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test-0", "_type": "user"}}'
- '{"ip": "10.0.0.1", "integer": 38, "float": 12.5713, "name": "Ruth", "bool": true}'
- '{"index": {"_index": "test-0", "_type": "user"}}'
- '{"ip": "10.0.0.2", "integer": 42, "float": 15.3393, "name": "Jackie", "surname": "Bowling", "bool": false}'
- '{"index": {"_index": "test-1", "_type": "user"}}'
- '{"ip": "10.0.0.3", "integer": 29, "float": 19.0517, "name": "Stephanie", "bool": true}'
- '{"index": {"_index": "test-1", "_type": "user"}}'
- '{"ip": "10.0.0.4", "integer": 19, "float": 19.3717, "surname": "Hamilton", "bool": true}'
- '{"index": {"_index": "test-2", "_type": "user"}}'
- '{"ip": "10.0.0.5", "integer": 0, "float": 17.3349, "name": "Natalie", "bool": false}'
---
"Search template with typed_keys parameter":
- do:
put_template:
id: template_1
body:
template:
query:
match:
bool: "{{bool_value}}"
aggs:
test_missing:
missing:
field: "{{missing_field}}"
- match: { acknowledged: true }
- do:
search_template:
index: test-*
typed_keys: true
body:
id: template_1
params:
bool_value: true
missing_field: name
- match: { hits.total: 3 }
- match: { aggregations.missing#test_missing.doc_count: 1 }
---
"Multisearch template with typed_keys parameter":
- do:
put_template:
id: registered_template
body:
template:
query:
range:
integer:
gte: "{{starting_value}}"
aggs:
test_histogram:
histogram:
field: "{{histo.field}}"
interval: "{{histo.interval}}"
- match: { acknowledged: true }
- do:
msearch_template:
typed_keys: true
body:
- index: test-*
- inline:
query:
match:
name: "{{name_value}}"
aggs:
test_global:
global: {}
aggs:
test_ip_range:
ip_range:
field: "{{ip_field}}"
ranges:
to: "10.0.0.6"
params:
name_value: "Stephanie"
ip_field: "ip"
- index: test-0,test-1
- id: registered_template
params:
starting_value: 30
histo:
field: float
interval: 5
- match: { responses.0.hits.total: 1 }
- match: { responses.0.aggregations.global#test_global.doc_count: 5 }
- match: { responses.0.aggregations.global#test_global.ip_range#test_ip_range.buckets.0.doc_count: 5 }
- match: { responses.1.hits.total: 2 }
- match: { responses.1.aggregations.histogram#test_histogram.buckets.0.doc_count: 1 }

View File

@ -24,6 +24,10 @@
"max_concurrent_searches" : {
"type" : "number",
"description" : "Controls the maximum number of concurrent searches the multi search api will execute"
},
"typed_keys": {
"type" : "boolean",
"description" : "Specify whether aggregation names should be prefixed by their respective types in the response"
}
}
},

View File

@ -20,6 +20,10 @@
"type" : "enum",
"options" : ["query_then_fetch", "query_and_fetch", "dfs_query_then_fetch", "dfs_query_and_fetch"],
"description" : "Search operation type"
},
"typed_keys": {
"type" : "boolean",
"description" : "Specify whether aggregation names should be prefixed by their respective types in the response"
}
}
},

View File

@ -147,6 +147,10 @@
"type" : "boolean",
"description": "Whether to calculate and return scores even if they are not used for sorting"
},
"typed_keys": {
"type" : "boolean",
"description" : "Specify whether aggregation names should be prefixed by their respective types in the response"
},
"version": {
"type" : "boolean",
"description" : "Specify whether to return document version as part of a hit"

View File

@ -54,6 +54,10 @@
"profile": {
"type" : "boolean",
"description" : "Specify whether to profile the query execution"
},
"typed_keys": {
"type" : "boolean",
"description" : "Specify whether aggregation names should be prefixed by their respective types in the response"
}
}
},

View File

@ -0,0 +1,83 @@
setup:
- skip:
version: " - 5.3.99"
reason: typed_keys parameter was added in 5.4.0
- do:
indices.put_template:
name: index_template
body:
index_patterns: test-*
settings:
number_of_replicas: 0
mappings:
user:
properties:
index_start_at:
type: integer
integer:
type: integer
float:
type: float
name:
type: keyword
- do:
bulk:
refresh: true
body:
- '{"index": {"_index": "test-0", "_type": "user"}}'
- '{"row": 1, "index_start_at": 56, "integer": 38, "float": 12.5713, "name": "Ruth", "bool": true}'
- '{"index": {"_index": "test-0", "_type": "user"}}'
- '{"row": 2, "index_start_at": 57, "integer": 42, "float": 15.3393, "name": "Jackie", "surname": "Bowling", "bool": false}'
- '{"index": {"_index": "test-1", "_type": "user"}}'
- '{"row": 3, "index_start_at": 58, "integer": 29, "float": 19.0517, "name": "Stephanie", "bool": true}'
- '{"index": {"_index": "test-1", "_type": "user"}}'
- '{"row": 4, "index_start_at": 59, "integer": 19, "float": 19.3717, "surname": "Hamilton", "bool": true}'
- '{"index": {"_index": "test-2", "_type": "user"}}'
- '{"row": 5, "index_start_at": 60, "integer": 0, "float": 17.3349, "name": "Natalie", "bool": false}'
---
"Multisearch test with typed_keys parameter":
- do:
msearch:
typed_keys: true
body:
- index: test-*
- {query: {match: {bool: true} }, size: 0, aggs: {test_filter: {filter: {range:{integer: {gte: 20} } } } } }
- index: test-1
- {query: {match_all: {} }, size: 0, aggs: {test_range: {range: {field: float, ranges: [ {to: 19.2499999}, {from: 19.25} ] } } } }
- index: test-*
- {query: {bool: {filter: {range: {row: {lt: 5}}} } }, size: 0, aggs: {test_percentiles: {percentiles: {field: float} } } }
- match: { responses.0.hits.total: 3 }
- match: { responses.0.aggregations.filter#test_filter.doc_count : 2 }
- match: { responses.1.hits.total: 2 }
- match: { responses.1.aggregations.range#test_range.buckets.0.key : "*-19.2499999" }
- match: { responses.1.aggregations.range#test_range.buckets.0.doc_count : 1 }
- match: { responses.1.aggregations.range#test_range.buckets.1.key : "19.25-*" }
- match: { responses.1.aggregations.range#test_range.buckets.1.doc_count : 1 }
- match: { responses.2.hits.total: 4 }
- is_true: responses.2.aggregations.tdigest_percentiles#test_percentiles.values
---
"Multisearch test with typed_keys parameter for sampler and significant terms":
- do:
msearch:
typed_keys: true
body:
- index: test-*
- {query: {match_all: {} }, size: 0, aggs: {test_sampler: {sampler: {shard_size: 200}, aggs: {test_significant_terms: {significant_terms: {field: name} } } } } }
- index: test-*
- {query: {match_all: {} }, size: 0, aggs: {test_umterms: {terms: {field: surname} } } }
- index: test-*
- {query: {match_all: {} }, size: 0, aggs: {test_sterms: {terms: {field: name}, aggs: {test_umsignificant_terms: {significant_terms: {field: surname} } } } } }
- match: { responses.0.hits.total: 5 }
- match: { responses.0.aggregations.sampler#test_sampler.doc_count : 5 }
- match: { responses.0.aggregations.sampler#test_sampler.sigsterms#test_significant_terms.doc_count : 5 }
- match: { responses.1.hits.total: 1 }
- match: { responses.1.aggregations.sterms#test_umterms.doc_count_error_upper_bound : 0 }
- match: { responses.2.hits.total: 1 }
- match: { responses.2.aggregations.sterms#test_sterms.doc_count_error_upper_bound : 0 }
- is_true: responses.2.aggregations.sterms#test_sterms.buckets.0.sigsterms#test_umsignificant_terms

View File

@ -0,0 +1,226 @@
setup:
- skip:
version: " - 5.3.99"
reason: typed_keys parameter was added in 5.4.0
- do:
indices.create:
index: test
body:
settings:
number_of_replicas: 0
mappings:
test:
properties:
name:
type: keyword
num:
type: integer
created:
type: date
- do:
bulk:
refresh: true
index: test
type: test
body:
- '{"index": {}}'
- '{"name": "one", "num": 1, "created": "2010-03-12T01:07:45"}'
- '{"index": {}}'
- '{"name": "two", "num": 2, "created": "2010-03-12T04:11:00"}'
- '{"index": {}}'
- '{"name": "three", "num": 3, "created": "2010-04-27T03:43:34"}'
---
"Test typed keys parameter for avg aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_avg:
avg:
field: num
- is_true: aggregations.avg#test_avg
---
"Test typed keys parameter for cardinality aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_cardinality:
cardinality:
field: name
- is_true: aggregations.cardinality#test_cardinality
---
"Test typed keys parameter for extended_stats aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_extended_stats:
extended_stats:
field: num
- is_true: aggregations.extended_stats#test_extended_stats
---
"Test typed keys parameter for max aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_max:
max:
field: num
- is_true: aggregations.max#test_max
---
"Test typed keys parameter for min aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_min:
min:
field: num
- is_true: aggregations.min#test_min
---
"Test typed keys parameter for percentiles aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_percentiles:
percentiles:
field: num
- is_true: aggregations.tdigest_percentiles#test_percentiles
---
"Test typed keys parameter for percentile_ranks aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_percentile_ranks:
percentile_ranks:
field: num
values: [0,10]
- is_true: aggregations.tdigest_percentile_ranks#test_percentile_ranks
---
"Test typed keys parameter for stats aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_stats:
stats:
field: num
- is_true: aggregations.stats#test_stats
---
"Test typed keys parameter for sum aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_sum:
sum:
field: num
- is_true: aggregations.sum#test_sum
---
"Test typed keys parameter for terms and top_hits aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_terms:
terms:
field: name
aggs:
test_top_hits:
top_hits:
sort: num
- is_true: aggregations.sterms#test_terms
- is_true: aggregations.sterms#test_terms.buckets.0.top_hits#test_top_hits
- is_true: aggregations.sterms#test_terms.buckets.1.top_hits#test_top_hits
- is_true: aggregations.sterms#test_terms.buckets.2.top_hits#test_top_hits
---
"Test typed keys parameter for terms aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_terms:
terms:
field: num
- is_true: aggregations.lterms#test_terms
---
"Test typed keys parameter for value_count aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_value_count:
value_count:
field: num
- is_true: aggregations.value_count#test_value_count
---
"Test typed keys parameter for date_histogram aggregation and max_bucket pipeline aggregation":
- do:
search:
typed_keys: true
body:
size: 0
aggregations:
test_created_histogram:
date_histogram:
field: created
interval: month
aggregations:
test_sum:
sum:
field: num
test_moving_avg:
moving_avg:
buckets_path: "test_sum"
test_max_bucket:
max_bucket:
buckets_path: "test_created_histogram>test_sum"
- is_true: aggregations.date_histogram#test_created_histogram
- is_true: aggregations.date_histogram#test_created_histogram.buckets.0.sum#test_sum
- is_true: aggregations.date_histogram#test_created_histogram.buckets.1.sum#test_sum
- is_true: aggregations.date_histogram#test_created_histogram.buckets.1.simple_value#test_moving_avg
- is_true: aggregations.bucket_metric_value#test_max_bucket