honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support the 'fields' option in inner_hits and top_hits. (#62337) 

This PR adds support for the 'fields' option in the following places:
* Anytime `inner_hits` is used, for both fetching nested/ child docs and field collapsing
* The `top_hits` aggregation

Addresses #61949.
This commit is contained in:
Julie Tibshirani 2020-09-14 11:51:45 -07:00 committed by GitHub
parent 3d5c13f559
commit 4a19bdb2ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 299 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
View File

@ -24,6 +24,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
* <<highlighting,Highlighting>>
* <<request-body-search-explain,Explain>>
* <<named-queries,Named queries>>
* <<search-fields-param,Search fields>>
* <<source-filtering,Source filtering>>
* <<stored-fields,Stored fields>>
* <<script-fields,Script fields>>
@ -41,7 +42,7 @@ by aggregations, use a <<query-dsl-function-score-query,`function_score`>> or
==== Example
In the following example we group the sales by type and per type we show the last sale.
In the following example we group the sales by type and per type we show the last sale.
For each sale only the date and price fields are being included in the source.
[source,console]

1
docs/reference/search/search-your-data/retrieve-inner-hits.asciidoc
View File

@ -74,6 +74,7 @@ Inner hits also supports the following per document features:
* <<highlighting,Highlighting>>
* <<request-body-search-explain,Explain>>
* <<search-fields-param,Search fields>>
* <<request-body-search-source-filtering,Source filtering>>
* <<script-fields,Script fields>>
* <<docvalue-fields,Doc value fields>>

14
modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java
View File

@ -176,7 +176,7 @@ public class InnerHitsIT extends ParentChildTestCase {
.setQuery(
hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
new InnerHitBuilder()
.addDocValueField("message")
.addFetchField("message")
.setHighlightBuilder(new HighlightBuilder().field("message"))
.setExplain(true).setSize(1)
.addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5",
@ -187,8 +187,18 @@ public class InnerHitsIT extends ParentChildTestCase {
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(message:fox"));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
response = client().prepareSearch("articles")
.setQuery(
hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
new InnerHitBuilder().addDocValueField("message").setSize(1)
)).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
}
public void testRandomParentChild() throws Exception {

47
rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
View File

@ -6,6 +6,7 @@ setup:
mappings:
properties:
numeric_group: { type: integer }
tag: { type: keyword }
- do:
index:
@ -13,42 +14,42 @@ setup:
id: 1
version_type: external
version: 11
body: { numeric_group: 1, sort: 10 }
body: { numeric_group: 1, tag: A, sort: 10 }
- do:
index:
index: test
id: 2
version_type: external
version: 22
body: { numeric_group: 1, sort: 6 }
body: { numeric_group: 1, tag: B, sort: 6 }
- do:
index:
index: test
id: 3
version_type: external
version: 33
body: { numeric_group: 1, sort: 24 }
body: { numeric_group: 1, tag: A, sort: 24 }
- do:
index:
index: test
id: 4
version_type: external
version: 44
body: { numeric_group: 25, sort: 10 }
body: { numeric_group: 25, tag: B, sort: 10 }
- do:
index:
index: test
id: 5
version_type: external
version: 55
body: { numeric_group: 25, sort: 5 }
body: { numeric_group: 25, tag: A, sort: 5 }
- do:
index:
index: test
id: 6
version_type: external
version: 66
body: { numeric_group: 3, sort: 36 }
body: { numeric_group: 3, tag: B, sort: 36 }
- do:
indices.refresh:
index: test
@ -161,6 +162,40 @@ setup:
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
---
"field collapsing, inner_hits, and fields":
- skip:
version: " - 7.9.99"
reason: the fields option was added in 7.10
- do:
search:
rest_total_hits_as_int: true
index: test
body:
collapse:
field: numeric_group
inner_hits:
name: sub_hits
size: 2
sort: [{ sort: asc }]
fields: ["tag"]
sort: [{ sort: desc }]
- length: { hits.hits: 3 }
- length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
- match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
- match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
- length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1.fields.tag: ["A"]}
- length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0.fields.tag: ["A"]}
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1.fields.tag: ["B"]}
---
"field collapsing, inner_hits and maxConcurrentGroupRequests":

19
server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
View File

@ -166,6 +166,7 @@ public class TopHitsIT extends ESIntegTestCase {
.field(SORT_FIELD, i + 1)
.field("text", "some text to entertain")
.field("field1", 5)
.field("field2", 2.71)
.endObject()));
}
@ -315,7 +316,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat((Long) hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
assertThat((Long) hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
}
}
@ -402,7 +403,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(hits.getTotalHits().value, equalTo(10L));
assertThat(hits.getHits().length, equalTo(3));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
}
}
@ -433,7 +434,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(hits.getTotalHits().value, equalTo(10L));
assertThat(hits.getHits().length, equalTo(3));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
id--;
}
}
@ -597,6 +598,7 @@ public class TopHitsIT extends ESIntegTestCase {
.explain(true)
.storedField("text")
.docValueField("field1")
.fetchField("field2")
.scriptField("script",
new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
.fetchSource("text", null)
@ -639,13 +641,16 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(hit.getMatchedQueries()[0], equalTo("test"));
DocumentField field = hit.field("field1");
assertThat(field.getValue().toString(), equalTo("5"));
DocumentField field1 = hit.field("field1");
assertThat(field1.getValue(), equalTo(5L));
DocumentField field2 = hit.field("field2");
assertThat(field2.getValue(), equalTo(2.71f));
assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));
field = hit.field("script");
assertThat(field.getValue().toString(), equalTo("5"));
field2 = hit.field("script");
assertThat(field2.getValue().toString(), equalTo("5"));
assertThat(hit.getSourceAsMap().size(), equalTo(1));
assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));

14
server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
View File

@ -165,7 +165,7 @@ public class InnerHitsIT extends ESIntegTestCase {
.setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message"))
.setExplain(true)
.addDocValueField("comments.mes*")
.addFetchField("comments.mes*")
.addScriptField("script",
new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
.setSize(1))).get();
@ -176,8 +176,18 @@ public class InnerHitsIT extends ESIntegTestCase {
assertThat(innerHits.getAt(0).getHighlightFields().get("comments.message").getFragments()[0].string(),
equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(comments.message:fox in"));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
response = client().prepareSearch("articles")
.setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
new InnerHitBuilder()
.addDocValueField("comments.mes*")
.setSize(1))).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
}
public void testRandomNested() throws Exception {

3
server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
View File

@ -136,6 +136,9 @@ final class ExpandSearchPhase extends SearchPhase {
options.getFetchSourceContext().excludes());
}
}
if (options.getFetchFields() != null) {
options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format));
}
if (options.getDocValueFields() != null) {
options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format));
}

105
server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.query;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
@ -73,6 +74,8 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD);
PARSER.declareObjectArray(InnerHitBuilder::setDocValueFields,
(p,c) -> FieldAndFormat.fromXContent(p), SearchSourceBuilder.DOCVALUE_FIELDS_FIELD);
PARSER.declareObjectArray(InnerHitBuilder::setFetchFields,
(p,c) -> FieldAndFormat.fromXContent(p), SearchSourceBuilder.FETCH_FIELDS_FIELD);
PARSER.declareField((p, i, c) -> {
try {
Set<ScriptField> scriptFields = new HashSet<>();
@ -135,6 +138,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
private Set<ScriptField> scriptFields;
private HighlightBuilder highlightBuilder;
private FetchSourceContext fetchSourceContext;
private List<FieldAndFormat> fetchFields;
private CollapseBuilder innerCollapseBuilder = null;
public InnerHitBuilder() {
@ -195,6 +199,12 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
this.innerCollapseBuilder = in.readOptionalWriteable(CollapseBuilder::new);
}
if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
if (in.readBoolean()) {
fetchFields = in.readList(FieldAndFormat::new);
}
}
}
@Override
@ -243,6 +253,13 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
out.writeOptionalWriteable(innerCollapseBuilder);
}
if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
out.writeBoolean(fetchFields != null);
if (fetchFields != null) {
out.writeList(fetchFields);
}
}
}
public String getName() {
@ -379,6 +396,41 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
return addDocValueField(field, null);
}
/**
* Gets the fields to load and return as part of the search request.
*/
public List<FieldAndFormat> getFetchFields() {
return fetchFields;
}
/**
* Sets the stored fields to load and return as part of the search request.
*/
public InnerHitBuilder setFetchFields(List<FieldAndFormat> fetchFields) {
this.fetchFields = fetchFields;
return this;
}
/**
* Adds a field to load and return as part of the search request.
*/
public InnerHitBuilder addFetchField(String name) {
return addFetchField(name, null);
}
/**
* Adds a field to load and return as part of the search request.
* @param name the field name.
* @param format an optional format string used when formatting values, for example a date format.
*/
public InnerHitBuilder addFetchField(String name, @Nullable String format) {
if (fetchFields == null) {
fetchFields = new ArrayList<>();
}
fetchFields.add(new FieldAndFormat(name, format));
return this;
}
public Set<ScriptField> getScriptFields() {
return scriptFields;
}
@ -466,14 +518,14 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
if (docValueFields != null) {
builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat docValueField : docValueFields) {
if (docValueField.format == null) {
builder.value(docValueField.field);
} else {
builder.startObject()
.field("field", docValueField.field)
.field("format", docValueField.format)
.endObject();
}
docValueField.toXContent(builder, params);
}
builder.endArray();
}
if (fetchFields != null) {
builder.startArray(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat docValueField : fetchFields) {
docValueField.toXContent(builder, params);
}
builder.endArray();
}
@ -505,29 +557,30 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
InnerHitBuilder that = (InnerHitBuilder) o;
return Objects.equals(name, that.name) &&
Objects.equals(ignoreUnmapped, that.ignoreUnmapped) &&
Objects.equals(from, that.from) &&
Objects.equals(size, that.size) &&
Objects.equals(explain, that.explain) &&
Objects.equals(version, that.version) &&
Objects.equals(seqNoAndPrimaryTerm, that.seqNoAndPrimaryTerm) &&
Objects.equals(trackScores, that.trackScores) &&
Objects.equals(storedFieldsContext, that.storedFieldsContext) &&
Objects.equals(docValueFields, that.docValueFields) &&
Objects.equals(scriptFields, that.scriptFields) &&
Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
Objects.equals(sorts, that.sorts) &&
Objects.equals(highlightBuilder, that.highlightBuilder) &&
Objects.equals(innerCollapseBuilder, that.innerCollapseBuilder);
return ignoreUnmapped == that.ignoreUnmapped &&
from == that.from &&
size == that.size &&
explain == that.explain &&
version == that.version &&
seqNoAndPrimaryTerm == that.seqNoAndPrimaryTerm &&
trackScores == that.trackScores &&
Objects.equals(name, that.name) &&
Objects.equals(storedFieldsContext, that.storedFieldsContext) &&
Objects.equals(query, that.query) &&
Objects.equals(sorts, that.sorts) &&
Objects.equals(docValueFields, that.docValueFields) &&
Objects.equals(scriptFields, that.scriptFields) &&
Objects.equals(highlightBuilder, that.highlightBuilder) &&
Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
Objects.equals(fetchFields, that.fetchFields) &&
Objects.equals(innerCollapseBuilder, that.innerCollapseBuilder);
}
@Override
public int hashCode() {
return Objects.hash(name, ignoreUnmapped, from, size, explain, version, seqNoAndPrimaryTerm, trackScores,
storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, innerCollapseBuilder);
return Objects.hash(name, ignoreUnmapped, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, storedFieldsContext,
query, sorts, docValueFields, scriptFields, highlightBuilder, fetchSourceContext, fetchFields, innerCollapseBuilder);
}
public static InnerHitBuilder fromXContent(XContentParser parser) throws IOException {

7
server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.script.FieldScript;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext;
import org.elasticsearch.search.fetch.subphase.FetchFieldsContext;
import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
@ -92,6 +93,12 @@ public abstract class InnerHitContextBuilder {
queryShardContext.getMapperService(), innerHitBuilder.getDocValueFields());
innerHitsContext.docValuesContext(docValuesContext);
}
if (innerHitBuilder.getFetchFields() != null) {
String indexName = queryShardContext.index().getName();
FetchFieldsContext fieldsContext = FetchFieldsContext.create(
indexName, queryShardContext.getMapperService(), innerHitBuilder.getFetchFields());
innerHitsContext.fetchFieldsContext(fieldsContext);
}
if (innerHitBuilder.getScriptFields() != null) {
for (SearchSourceBuilder.ScriptField field : innerHitBuilder.getScriptFields()) {
QueryShardContext innerContext = innerHitsContext.getQueryShardContext();

122
server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
View File

@ -72,6 +72,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
private HighlightBuilder highlightBuilder;
private StoredFieldsContext storedFieldsContext;
private List<FieldAndFormat> docValueFields;
private List<FieldAndFormat> fetchFields;
private Set<ScriptField> scriptFields;
private FetchSourceContext fetchSourceContext;
@ -94,6 +95,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
this.storedFieldsContext = clone.storedFieldsContext == null ? null :
new StoredFieldsContext(clone.storedFieldsContext);
this.docValueFields = clone.docValueFields == null ? null : new ArrayList<>(clone.docValueFields);
this.fetchFields = clone.fetchFields == null ? null : new ArrayList<>(clone.fetchFields);
this.scriptFields = clone.scriptFields == null ? null : new HashSet<>(clone.scriptFields);
this.fetchSourceContext = clone.fetchSourceContext == null ? null :
new FetchSourceContext(clone.fetchSourceContext.fetchSource(), clone.fetchSourceContext.includes(),
@ -142,6 +144,12 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
if (in.getVersion().onOrAfter(Version.V_6_7_0)) {
seqNoAndPrimaryTerm = in.readBoolean();
}
if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
if (in.readBoolean()) {
fetchFields = in.readList(FieldAndFormat::new);
}
}
}
@Override
@ -172,6 +180,13 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
if (out.getVersion().onOrAfter(Version.V_6_7_0)) {
out.writeBoolean(seqNoAndPrimaryTerm);
}
if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
out.writeBoolean(fetchFields != null);
if (fetchFields != null) {
out.writeList(fetchFields);
}
}
}
/**
@ -430,10 +445,38 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
/**
* Gets the field-data fields.
*/
public List<FieldAndFormat> fieldDataFields() {
public List<FieldAndFormat> docValueFields() {
return docValueFields;
}
/**
* Adds a field to load and return as part of the search request.
*/
public TopHitsAggregationBuilder fetchField(String field, String format) {
if (field == null) {
throw new IllegalArgumentException("[fields] must not be null: [" + name + "]");
}
if (fetchFields == null) {
fetchFields = new ArrayList<>();
}
fetchFields.add(new FieldAndFormat(field, format));
return this;
}
/**
* Adds a field to load and return as part of the search request.
*/
public TopHitsAggregationBuilder fetchField(String field) {
return fetchField(field, null);
}
/**
* Gets the fields to load and return as part of the search request.
*/
public List<FieldAndFormat> fetchFields() {
return fetchFields;
}
/**
* Adds a script field under the given name with the provided script.
*
@ -585,12 +628,12 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
);
}
List<ScriptFieldsContext.ScriptField> fields = new ArrayList<>();
if (scriptFields != null) {
for (ScriptField field : scriptFields) {
List<ScriptFieldsContext.ScriptField> scriptFields = new ArrayList<>();
if (this.scriptFields != null) {
for (ScriptField field : this.scriptFields) {
FieldScript.Factory factory = queryShardContext.compile(field.script(), FieldScript.CONTEXT);
FieldScript.LeafFactory searchScript = factory.newFactory(field.script().getParams(), queryShardContext.lookup());
fields.add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
scriptFields.add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
field.fieldName(), searchScript, field.ignoreFailure()));
}
}
@ -602,7 +645,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
optionalSort = SortBuilder.buildSort(sorts, queryShardContext);
}
return new TopHitsAggregatorFactory(name, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, optionalSort,
highlightBuilder, storedFieldsContext, docValueFields, fields, fetchSourceContext, queryShardContext, parent,
highlightBuilder, storedFieldsContext, docValueFields, fetchFields, scriptFields, fetchSourceContext, queryShardContext, parent,
subfactoriesBuilder, metadata);
}
@ -620,18 +663,23 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
if (storedFieldsContext != null) {
storedFieldsContext.toXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), builder);
}
if (docValueFields != null) {
builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat dvField : docValueFields) {
builder.startObject()
.field("field", dvField.field);
if (dvField.format != null) {
builder.field("format", dvField.format);
}
builder.endObject();
for (FieldAndFormat docValueField : docValueFields) {
docValueField.toXContent(builder, params);
}
builder.endArray();
}
if (fetchFields != null) {
builder.startArray(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat docValueField : fetchFields) {
docValueField.toXContent(builder, params);
}
builder.endArray();
}
if (scriptFields != null) {
builder.startObject(SearchSourceBuilder.SCRIPT_FIELDS_FIELD.getPreferredName());
for (ScriptField scriptField : scriptFields) {
@ -751,6 +799,11 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
FieldAndFormat ff = FieldAndFormat.fromXContent(parser);
factory.docValueField(ff.field, ff.format);
}
} else if (SearchSourceBuilder.FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
FieldAndFormat ff = FieldAndFormat.fromXContent(parser);
factory.fetchField(ff.field, ff.format);
}
} else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
List<SortBuilder<?>> sorts = SortBuilder.fromXContent(parser);
factory.sorts(sorts);
@ -769,31 +822,30 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), explain, fetchSourceContext, docValueFields,
storedFieldsContext, from, highlightBuilder,
scriptFields, size, sorts, trackScores, version,
seqNoAndPrimaryTerm);
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
TopHitsAggregationBuilder that = (TopHitsAggregationBuilder) o;
return from == that.from &&
size == that.size &&
explain == that.explain &&
version == that.version &&
seqNoAndPrimaryTerm == that.seqNoAndPrimaryTerm &&
trackScores == that.trackScores &&
Objects.equals(sorts, that.sorts) &&
Objects.equals(highlightBuilder, that.highlightBuilder) &&
Objects.equals(storedFieldsContext, that.storedFieldsContext) &&
Objects.equals(docValueFields, that.docValueFields) &&
Objects.equals(fetchFields, that.fetchFields) &&
Objects.equals(scriptFields, that.scriptFields) &&
Objects.equals(fetchSourceContext, that.fetchSourceContext);
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null || getClass() != obj.getClass()) return false;
if (super.equals(obj) == false) return false;
TopHitsAggregationBuilder other = (TopHitsAggregationBuilder) obj;
return Objects.equals(explain, other.explain)
&& Objects.equals(fetchSourceContext, other.fetchSourceContext)
&& Objects.equals(docValueFields, other.docValueFields)
&& Objects.equals(storedFieldsContext, other.storedFieldsContext)
&& Objects.equals(from, other.from)
&& Objects.equals(highlightBuilder, other.highlightBuilder)
&& Objects.equals(scriptFields, other.scriptFields)
&& Objects.equals(size, other.size)
&& Objects.equals(sorts, other.sorts)
&& Objects.equals(trackScores, other.trackScores)
&& Objects.equals(version, other.version)
&& Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm);
public int hashCode() {
return Objects.hash(super.hashCode(), from, size, explain, version, seqNoAndPrimaryTerm, trackScores, sorts, highlightBuilder,
storedFieldsContext, docValueFields, fetchFields, scriptFields, fetchSourceContext);
}
@Override

9
server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
import org.elasticsearch.search.aggregations.CardinalityUpperBound;
import org.elasticsearch.search.fetch.StoredFieldsContext;
import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext;
import org.elasticsearch.search.fetch.subphase.FetchFieldsContext;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
@ -51,6 +52,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
private final HighlightBuilder highlightBuilder;
private final StoredFieldsContext storedFieldsContext;
private final List<FieldAndFormat> docValueFields;
private final List<FieldAndFormat> fetchFields;
private final List<ScriptFieldsContext.ScriptField> scriptFields;
private final FetchSourceContext fetchSourceContext;
@ -65,6 +67,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
HighlightBuilder highlightBuilder,
StoredFieldsContext storedFieldsContext,
List<FieldAndFormat> docValueFields,
List<FieldAndFormat> fetchFields,
List<ScriptFieldsContext.ScriptField> scriptFields,
FetchSourceContext fetchSourceContext,
QueryShardContext queryShardContext,
@ -82,6 +85,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
this.highlightBuilder = highlightBuilder;
this.storedFieldsContext = storedFieldsContext;
this.docValueFields = docValueFields;
this.fetchFields = fetchFields;
this.scriptFields = scriptFields;
this.fetchSourceContext = fetchSourceContext;
}
@ -109,6 +113,11 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
FetchDocValuesContext docValuesContext = FetchDocValuesContext.create(searchContext.mapperService(), docValueFields);
subSearchContext.docValuesContext(docValuesContext);
}
if (fetchFields != null) {
String indexName = searchContext.indexShard().shardId().getIndexName();
FetchFieldsContext fieldsContext = FetchFieldsContext.create(indexName, searchContext.mapperService(), fetchFields);
subSearchContext.fetchFieldsContext(fieldsContext);
}
for (ScriptFieldsContext.ScriptField field : scriptFields) {
subSearchContext.scriptFields().add(field);
}

10
server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View File

@ -172,7 +172,7 @@ public class FetchPhase {
if (!context.hasScriptFields() && !context.hasFetchSourceContext()) {
context.fetchSourceContext(new FetchSourceContext(true));
}
boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null;
boolean loadSource = sourceRequired(context);
return new FieldsVisitor(loadSource);
} else if (storedFieldsContext.fetchFields() == false) {
// disable stored fields entirely
@ -202,7 +202,7 @@ public class FetchPhase {
}
}
}
boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null;
boolean loadSource = sourceRequired(context);
if (storedToRequestedFields.isEmpty()) {
// empty list specified, default to disable _source if no explicit indication
return new FieldsVisitor(loadSource);
@ -212,6 +212,10 @@ public class FetchPhase {
}
}
private boolean sourceRequired(SearchContext context) {
return context.sourceRequested() || context.fetchFieldsContext() != null;
}
private int findRootDocumentIfNested(SearchContext context, LeafReaderContext subReaderContext, int subDocId) throws IOException {
if (context.mapperService().hasNested()) {
BitSet bits = context.bitsetFilterCache()
@ -295,7 +299,7 @@ public class FetchPhase {
// Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
// because the entire _source is only stored with the root document.
boolean needSource = context.sourceRequested() || context.highlight() != null;
boolean needSource = sourceRequired(context) || context.highlight() != null;
Uid rootId;
Map<String, Object> rootSourceAsMap = null;

19
server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
View File

@ -158,6 +158,8 @@ public class InnerHitBuilderTests extends ESTestCase {
}
innerHits.setDocValueFields(randomListStuff(16,
() -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)));
innerHits.setFetchFields(randomListStuff(16,
() -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)));
// Random script fields deduped on their field name.
Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
@ -204,6 +206,14 @@ public class InnerHitBuilderTests extends ESTestCase {
copy.addDocValueField(randomAlphaOfLengthBetween(1, 16));
}
});
modifiers.add(() -> {
if (randomBoolean()) {
copy.setFetchFields(randomValueOtherThan(copy.getFetchFields(),
() -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null))));
} else {
copy.addFetchField(randomAlphaOfLengthBetween(1, 16));
}
});
modifiers.add(() -> {
if (randomBoolean()) {
copy.setScriptFields(randomValueOtherThan(copy.getScriptFields(), () -> {
@ -292,4 +302,13 @@ public class InnerHitBuilderTests extends ESTestCase {
Arrays.asList(new FieldAndFormat("foo", null), new FieldAndFormat("@timestamp", "epoch_millis")),
innerHit.getDocValueFields());
}
public void testSetFetchFieldFormat() {
InnerHitBuilder innerHit = new InnerHitBuilder();
innerHit.addFetchField("foo");
innerHit.addFetchField("@timestamp", "epoch_millis");
assertEquals(
Arrays.asList(new FieldAndFormat("foo", null), new FieldAndFormat("@timestamp", "epoch_millis")),
innerHit.getFetchFields());
}
}

6
server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
View File

@ -86,6 +86,12 @@ public class TopHitsTests extends BaseAggregationTestCase<TopHitsAggregationBuil
factory.docValueField(randomAlphaOfLengthBetween(5, 50));
}
}
if (randomBoolean()) {
int fetchFieldsSize = randomInt(25);
for (int i = 0; i < fetchFieldsSize; i++) {
factory.fetchField(randomAlphaOfLengthBetween(5, 50));
}
}
if (randomBoolean()) {
int scriptFieldsSize = randomInt(25);
for (int i = 0; i < scriptFieldsSize; i++) {

3
x-pack/plugin/runtime-fields/qa/rest/build.gradle
View File

@ -32,7 +32,8 @@ yamlRestTest {
systemProperty 'tests.rest.blacklist',
[
/////// TO FIX ///////
'search/330_fetch_fields/*', // The whole API is not yet supported
'search/330_fetch_fields/*', // The 'fields' option is not yet supported
'search/110_field_collapsing/field collapsing, inner_hits, and fields', // Also fails because of the 'fields' option
'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing?
'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix.
'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do?