honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support the 'fields' option in inner_hits and top_hits. (#62337) 

This PR adds support for the 'fields' option in the following places:
* Anytime `inner_hits` is used, for both fetching nested/ child docs and field collapsing
* The `top_hits` aggregation

Addresses #61949.
This commit is contained in:
Julie Tibshirani 2020-09-14 11:51:45 -07:00 committed by GitHub
parent 3d5c13f559
commit 4a19bdb2ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 299 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
View File

@ -24,6 +24,7 @@ The top_hits aggregation returns regular search hits, because of this many per h
* <<highlighting,Highlighting>>
* <<request-body-search-explain,Explain>>
* <<named-queries,Named queries>>
* <<search-fields-param,Search fields>>
* <<source-filtering,Source filtering>>
* <<stored-fields,Stored fields>>
* <<script-fields,Script fields>>

1
docs/reference/search/search-your-data/retrieve-inner-hits.asciidoc
View File

@ -74,6 +74,7 @@ Inner hits also supports the following per document features:
* <<highlighting,Highlighting>>
* <<request-body-search-explain,Explain>>
* <<search-fields-param,Search fields>>
* <<request-body-search-source-filtering,Source filtering>>
* <<script-fields,Script fields>>
* <<docvalue-fields,Doc value fields>>

14
modules/parent-join/src/internalClusterTest/java/org/elasticsearch/join/query/InnerHitsIT.java
View File

@ -176,7 +176,7 @@ public class InnerHitsIT extends ParentChildTestCase {
.setQuery(
hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
new InnerHitBuilder()
.addDocValueField("message")
.addFetchField("message")
.setHighlightBuilder(new HighlightBuilder().field("message"))
.setExplain(true).setSize(1)
.addScriptField("script", new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5",
@ -187,8 +187,18 @@ public class InnerHitsIT extends ParentChildTestCase {
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getHighlightFields().get("message").getFragments()[0].string(), equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(message:fox"));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
response = client().prepareSearch("articles")
.setQuery(
hasChildQuery("comment", matchQuery("message", "fox"), ScoreMode.None).innerHit(
new InnerHitBuilder().addDocValueField("message").setSize(1)
)).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comment");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("message").getValue().toString(), equalTo("eat"));
}
public void testRandomParentChild() throws Exception {

47
rest-api-spec/src/main/resources/rest-api-spec/test/search/110_field_collapsing.yml
View File

@ -6,6 +6,7 @@ setup:
mappings:
properties:
numeric_group: { type: integer }
tag: { type: keyword }
- do:
index:
@ -13,42 +14,42 @@ setup:
id: 1
version_type: external
version: 11
body: { numeric_group: 1, sort: 10 }
body: { numeric_group: 1, tag: A, sort: 10 }
- do:
index:
index: test
id: 2
version_type: external
version: 22
body: { numeric_group: 1, sort: 6 }
body: { numeric_group: 1, tag: B, sort: 6 }
- do:
index:
index: test
id: 3
version_type: external
version: 33
body: { numeric_group: 1, sort: 24 }
body: { numeric_group: 1, tag: A, sort: 24 }
- do:
index:
index: test
id: 4
version_type: external
version: 44
body: { numeric_group: 25, sort: 10 }
body: { numeric_group: 25, tag: B, sort: 10 }
- do:
index:
index: test
id: 5
version_type: external
version: 55
body: { numeric_group: 25, sort: 5 }
body: { numeric_group: 25, tag: A, sort: 5 }
- do:
index:
index: test
id: 6
version_type: external
version: 66
body: { numeric_group: 3, sort: 36 }
body: { numeric_group: 3, tag: B, sort: 36 }
- do:
indices.refresh:
index: test
@ -161,6 +162,40 @@ setup:
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
---
"field collapsing, inner_hits, and fields":
- skip:
version: " - 7.9.99"
reason: the fields option was added in 7.10
- do:
search:
rest_total_hits_as_int: true
index: test
body:
collapse:
field: numeric_group
inner_hits:
name: sub_hits
size: 2
sort: [{ sort: asc }]
fields: ["tag"]
sort: [{ sort: desc }]
- length: { hits.hits: 3 }
- length: { hits.hits.0.inner_hits.sub_hits.hits.hits: 1 }
- match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0._id: "6" }
- match: { hits.hits.0.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
- length: { hits.hits.1.inner_hits.sub_hits.hits.hits: 2 }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0._id: "2" }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.0.fields.tag: ["B"]}
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1._id: "1" }
- match: { hits.hits.1.inner_hits.sub_hits.hits.hits.1.fields.tag: ["A"]}
- length: { hits.hits.2.inner_hits.sub_hits.hits.hits: 2 }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0._id: "5" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.0.fields.tag: ["A"]}
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1._id: "4" }
- match: { hits.hits.2.inner_hits.sub_hits.hits.hits.1.fields.tag: ["B"]}
---
"field collapsing, inner_hits and maxConcurrentGroupRequests":

19
server/src/internalClusterTest/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
View File

@ -166,6 +166,7 @@ public class TopHitsIT extends ESIntegTestCase {
.field(SORT_FIELD, i + 1)
.field("text", "some text to entertain")
.field("field1", 5)
.field("field2", 2.71)
.endObject()));
}
@ -315,7 +316,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat((Long) hits.getAt(1).getSortValues()[0], equalTo(higestSortValue - 1));
assertThat((Long) hits.getAt(2).getSortValues()[0], equalTo(higestSortValue - 2));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
}
}
@ -402,7 +403,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(hits.getTotalHits().value, equalTo(10L));
assertThat(hits.getHits().length, equalTo(3));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
}
}
@ -433,7 +434,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(hits.getTotalHits().value, equalTo(10L));
assertThat(hits.getHits().length, equalTo(3));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(4));
assertThat(hits.getAt(0).getSourceAsMap().size(), equalTo(5));
id--;
}
}
@ -597,6 +598,7 @@ public class TopHitsIT extends ESIntegTestCase {
.explain(true)
.storedField("text")
.docValueField("field1")
.fetchField("field2")
.scriptField("script",
new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
.fetchSource("text", null)
@ -639,13 +641,16 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(hit.getMatchedQueries()[0], equalTo("test"));
DocumentField field = hit.field("field1");
assertThat(field.getValue().toString(), equalTo("5"));
DocumentField field1 = hit.field("field1");
assertThat(field1.getValue(), equalTo(5L));
DocumentField field2 = hit.field("field2");
assertThat(field2.getValue(), equalTo(2.71f));
assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));
field = hit.field("script");
assertThat(field.getValue().toString(), equalTo("5"));
field2 = hit.field("script");
assertThat(field2.getValue().toString(), equalTo("5"));
assertThat(hit.getSourceAsMap().size(), equalTo(1));
assertThat(hit.getSourceAsMap().get("text").toString(), equalTo("some text to entertain"));

14
server/src/internalClusterTest/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
View File

@ -165,7 +165,7 @@ public class InnerHitsIT extends ESIntegTestCase {
.setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
new InnerHitBuilder().setHighlightBuilder(new HighlightBuilder().field("comments.message"))
.setExplain(true)
.addDocValueField("comments.mes*")
.addFetchField("comments.mes*")
.addScriptField("script",
new Script(ScriptType.INLINE, MockScriptEngine.NAME, "5", Collections.emptyMap()))
.setSize(1))).get();
@ -176,8 +176,18 @@ public class InnerHitsIT extends ESIntegTestCase {
assertThat(innerHits.getAt(0).getHighlightFields().get("comments.message").getFragments()[0].string(),
equalTo("<em>fox</em> eat quick"));
assertThat(innerHits.getAt(0).getExplanation().toString(), containsString("weight(comments.message:fox in"));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("fox eat quick"));
assertThat(innerHits.getAt(0).getFields().get("script").getValue().toString(), equalTo("5"));
response = client().prepareSearch("articles")
.setQuery(nestedQuery("comments", matchQuery("comments.message", "fox"), ScoreMode.Avg).innerHit(
new InnerHitBuilder()
.addDocValueField("comments.mes*")
.setSize(1))).get();
assertNoFailures(response);
innerHits = response.getHits().getAt(0).getInnerHits().get("comments");
assertThat(innerHits.getHits().length, equalTo(1));
assertThat(innerHits.getAt(0).getFields().get("comments.message").getValue().toString(), equalTo("eat"));
}
public void testRandomNested() throws Exception {

3
server/src/main/java/org/elasticsearch/action/search/ExpandSearchPhase.java
View File

@ -136,6 +136,9 @@ final class ExpandSearchPhase extends SearchPhase {
options.getFetchSourceContext().excludes());
}
}
if (options.getFetchFields() != null) {
options.getFetchFields().forEach(ff -> groupSource.fetchField(ff.field, ff.format));
}
if (options.getDocValueFields() != null) {
options.getDocValueFields().forEach(ff -> groupSource.docValueField(ff.field, ff.format));
}

93
server/src/main/java/org/elasticsearch/index/query/InnerHitBuilder.java
View File

@ -19,6 +19,7 @@
package org.elasticsearch.index.query;
import org.elasticsearch.Version;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
@ -73,6 +74,8 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
PARSER.declareStringArray(InnerHitBuilder::setStoredFieldNames, SearchSourceBuilder.STORED_FIELDS_FIELD);
PARSER.declareObjectArray(InnerHitBuilder::setDocValueFields,
(p,c) -> FieldAndFormat.fromXContent(p), SearchSourceBuilder.DOCVALUE_FIELDS_FIELD);
PARSER.declareObjectArray(InnerHitBuilder::setFetchFields,
(p,c) -> FieldAndFormat.fromXContent(p), SearchSourceBuilder.FETCH_FIELDS_FIELD);
PARSER.declareField((p, i, c) -> {
try {
Set<ScriptField> scriptFields = new HashSet<>();
@ -135,6 +138,7 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
private Set<ScriptField> scriptFields;
private HighlightBuilder highlightBuilder;
private FetchSourceContext fetchSourceContext;
private List<FieldAndFormat> fetchFields;
private CollapseBuilder innerCollapseBuilder = null;
public InnerHitBuilder() {
@ -195,6 +199,12 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
if (in.getVersion().onOrAfter(Version.V_6_4_0)) {
this.innerCollapseBuilder = in.readOptionalWriteable(CollapseBuilder::new);
}
if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
if (in.readBoolean()) {
fetchFields = in.readList(FieldAndFormat::new);
}
}
}
@Override
@ -243,6 +253,13 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
if (out.getVersion().onOrAfter(Version.V_6_4_0)) {
out.writeOptionalWriteable(innerCollapseBuilder);
}
if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
out.writeBoolean(fetchFields != null);
if (fetchFields != null) {
out.writeList(fetchFields);
}
}
}
public String getName() {
@ -379,6 +396,41 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
return addDocValueField(field, null);
}
/**
* Gets the fields to load and return as part of the search request.
*/
public List<FieldAndFormat> getFetchFields() {
return fetchFields;
}
/**
* Sets the stored fields to load and return as part of the search request.
*/
public InnerHitBuilder setFetchFields(List<FieldAndFormat> fetchFields) {
this.fetchFields = fetchFields;
return this;
}
/**
* Adds a field to load and return as part of the search request.
*/
public InnerHitBuilder addFetchField(String name) {
return addFetchField(name, null);
}
/**
* Adds a field to load and return as part of the search request.
* @param name the field name.
* @param format an optional format string used when formatting values, for example a date format.
*/
public InnerHitBuilder addFetchField(String name, @Nullable String format) {
if (fetchFields == null) {
fetchFields = new ArrayList<>();
}
fetchFields.add(new FieldAndFormat(name, format));
return this;
}
public Set<ScriptField> getScriptFields() {
return scriptFields;
}
@ -466,14 +518,14 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
if (docValueFields != null) {
builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat docValueField : docValueFields) {
if (docValueField.format == null) {
builder.value(docValueField.field);
} else {
builder.startObject()
.field("field", docValueField.field)
.field("format", docValueField.format)
.endObject();
docValueField.toXContent(builder, params);
}
builder.endArray();
}
if (fetchFields != null) {
builder.startArray(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat docValueField : fetchFields) {
docValueField.toXContent(builder, params);
}
builder.endArray();
}
@ -505,29 +557,30 @@ public final class InnerHitBuilder implements Writeable, ToXContentObject {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
InnerHitBuilder that = (InnerHitBuilder) o;
return Objects.equals(name, that.name) &&
Objects.equals(ignoreUnmapped, that.ignoreUnmapped) &&
Objects.equals(from, that.from) &&
Objects.equals(size, that.size) &&
Objects.equals(explain, that.explain) &&
Objects.equals(version, that.version) &&
Objects.equals(seqNoAndPrimaryTerm, that.seqNoAndPrimaryTerm) &&
Objects.equals(trackScores, that.trackScores) &&
return ignoreUnmapped == that.ignoreUnmapped &&
from == that.from &&
size == that.size &&
explain == that.explain &&
version == that.version &&
seqNoAndPrimaryTerm == that.seqNoAndPrimaryTerm &&
trackScores == that.trackScores &&
Objects.equals(name, that.name) &&
Objects.equals(storedFieldsContext, that.storedFieldsContext) &&
Objects.equals(query, that.query) &&
Objects.equals(sorts, that.sorts) &&
Objects.equals(docValueFields, that.docValueFields) &&
Objects.equals(scriptFields, that.scriptFields) &&
Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
Objects.equals(sorts, that.sorts) &&
Objects.equals(highlightBuilder, that.highlightBuilder) &&
Objects.equals(fetchSourceContext, that.fetchSourceContext) &&
Objects.equals(fetchFields, that.fetchFields) &&
Objects.equals(innerCollapseBuilder, that.innerCollapseBuilder);
}
@Override
public int hashCode() {
return Objects.hash(name, ignoreUnmapped, from, size, explain, version, seqNoAndPrimaryTerm, trackScores,
storedFieldsContext, docValueFields, scriptFields, fetchSourceContext, sorts, highlightBuilder, innerCollapseBuilder);
return Objects.hash(name, ignoreUnmapped, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, storedFieldsContext,
query, sorts, docValueFields, scriptFields, highlightBuilder, fetchSourceContext, fetchFields, innerCollapseBuilder);
}
public static InnerHitBuilder fromXContent(XContentParser parser) throws IOException {

7
server/src/main/java/org/elasticsearch/index/query/InnerHitContextBuilder.java
View File

@ -23,6 +23,7 @@ import org.elasticsearch.index.IndexSettings;
import org.elasticsearch.script.FieldScript;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext;
import org.elasticsearch.search.fetch.subphase.FetchFieldsContext;
import org.elasticsearch.search.fetch.subphase.InnerHitsContext;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.sort.SortAndFormats;
@ -92,6 +93,12 @@ public abstract class InnerHitContextBuilder {
queryShardContext.getMapperService(), innerHitBuilder.getDocValueFields());
innerHitsContext.docValuesContext(docValuesContext);
}
if (innerHitBuilder.getFetchFields() != null) {
String indexName = queryShardContext.index().getName();
FetchFieldsContext fieldsContext = FetchFieldsContext.create(
indexName, queryShardContext.getMapperService(), innerHitBuilder.getFetchFields());
innerHitsContext.fetchFieldsContext(fieldsContext);
}
if (innerHitBuilder.getScriptFields() != null) {
for (SearchSourceBuilder.ScriptField field : innerHitBuilder.getScriptFields()) {
QueryShardContext innerContext = innerHitsContext.getQueryShardContext();

122
server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregationBuilder.java
View File

@ -72,6 +72,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
private HighlightBuilder highlightBuilder;
private StoredFieldsContext storedFieldsContext;
private List<FieldAndFormat> docValueFields;
private List<FieldAndFormat> fetchFields;
private Set<ScriptField> scriptFields;
private FetchSourceContext fetchSourceContext;
@ -94,6 +95,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
this.storedFieldsContext = clone.storedFieldsContext == null ? null :
new StoredFieldsContext(clone.storedFieldsContext);
this.docValueFields = clone.docValueFields == null ? null : new ArrayList<>(clone.docValueFields);
this.fetchFields = clone.fetchFields == null ? null : new ArrayList<>(clone.fetchFields);
this.scriptFields = clone.scriptFields == null ? null : new HashSet<>(clone.scriptFields);
this.fetchSourceContext = clone.fetchSourceContext == null ? null :
new FetchSourceContext(clone.fetchSourceContext.fetchSource(), clone.fetchSourceContext.includes(),
@ -142,6 +144,12 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
if (in.getVersion().onOrAfter(Version.V_6_7_0)) {
seqNoAndPrimaryTerm = in.readBoolean();
}
if (in.getVersion().onOrAfter(Version.V_7_10_0)) {
if (in.readBoolean()) {
fetchFields = in.readList(FieldAndFormat::new);
}
}
}
@Override
@ -172,6 +180,13 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
if (out.getVersion().onOrAfter(Version.V_6_7_0)) {
out.writeBoolean(seqNoAndPrimaryTerm);
}
if (out.getVersion().onOrAfter(Version.V_7_10_0)) {
out.writeBoolean(fetchFields != null);
if (fetchFields != null) {
out.writeList(fetchFields);
}
}
}
/**
@ -430,10 +445,38 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
/**
* Gets the field-data fields.
*/
public List<FieldAndFormat> fieldDataFields() {
public List<FieldAndFormat> docValueFields() {
return docValueFields;
}
/**
* Adds a field to load and return as part of the search request.
*/
public TopHitsAggregationBuilder fetchField(String field, String format) {
if (field == null) {
throw new IllegalArgumentException("[fields] must not be null: [" + name + "]");
}
if (fetchFields == null) {
fetchFields = new ArrayList<>();
}
fetchFields.add(new FieldAndFormat(field, format));
return this;
}
/**
* Adds a field to load and return as part of the search request.
*/
public TopHitsAggregationBuilder fetchField(String field) {
return fetchField(field, null);
}
/**
* Gets the fields to load and return as part of the search request.
*/
public List<FieldAndFormat> fetchFields() {
return fetchFields;
}
/**
* Adds a script field under the given name with the provided script.
*
@ -585,12 +628,12 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
);
}
List<ScriptFieldsContext.ScriptField> fields = new ArrayList<>();
if (scriptFields != null) {
for (ScriptField field : scriptFields) {
List<ScriptFieldsContext.ScriptField> scriptFields = new ArrayList<>();
if (this.scriptFields != null) {
for (ScriptField field : this.scriptFields) {
FieldScript.Factory factory = queryShardContext.compile(field.script(), FieldScript.CONTEXT);
FieldScript.LeafFactory searchScript = factory.newFactory(field.script().getParams(), queryShardContext.lookup());
fields.add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
scriptFields.add(new org.elasticsearch.search.fetch.subphase.ScriptFieldsContext.ScriptField(
field.fieldName(), searchScript, field.ignoreFailure()));
}
}
@ -602,7 +645,7 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
optionalSort = SortBuilder.buildSort(sorts, queryShardContext);
}
return new TopHitsAggregatorFactory(name, from, size, explain, version, seqNoAndPrimaryTerm, trackScores, optionalSort,
highlightBuilder, storedFieldsContext, docValueFields, fields, fetchSourceContext, queryShardContext, parent,
highlightBuilder, storedFieldsContext, docValueFields, fetchFields, scriptFields, fetchSourceContext, queryShardContext, parent,
subfactoriesBuilder, metadata);
}
@ -620,18 +663,23 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
if (storedFieldsContext != null) {
storedFieldsContext.toXContent(SearchSourceBuilder.STORED_FIELDS_FIELD.getPreferredName(), builder);
}
if (docValueFields != null) {
builder.startArray(SearchSourceBuilder.DOCVALUE_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat dvField : docValueFields) {
builder.startObject()
.field("field", dvField.field);
if (dvField.format != null) {
builder.field("format", dvField.format);
}
builder.endObject();
for (FieldAndFormat docValueField : docValueFields) {
docValueField.toXContent(builder, params);
}
builder.endArray();
}
if (fetchFields != null) {
builder.startArray(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName());
for (FieldAndFormat docValueField : fetchFields) {
docValueField.toXContent(builder, params);
}
builder.endArray();
}
if (scriptFields != null) {
builder.startObject(SearchSourceBuilder.SCRIPT_FIELDS_FIELD.getPreferredName());
for (ScriptField scriptField : scriptFields) {
@ -751,6 +799,11 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
FieldAndFormat ff = FieldAndFormat.fromXContent(parser);
factory.docValueField(ff.field, ff.format);
}
} else if (SearchSourceBuilder.FETCH_FIELDS_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
FieldAndFormat ff = FieldAndFormat.fromXContent(parser);
factory.fetchField(ff.field, ff.format);
}
} else if (SearchSourceBuilder.SORT_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
List<SortBuilder<?>> sorts = SortBuilder.fromXContent(parser);
factory.sorts(sorts);
@ -769,31 +822,30 @@ public class TopHitsAggregationBuilder extends AbstractAggregationBuilder<TopHit
}
@Override
public int hashCode() {
return Objects.hash(super.hashCode(), explain, fetchSourceContext, docValueFields,
storedFieldsContext, from, highlightBuilder,
scriptFields, size, sorts, trackScores, version,
seqNoAndPrimaryTerm);
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
if (!super.equals(o)) return false;
TopHitsAggregationBuilder that = (TopHitsAggregationBuilder) o;
return from == that.from &&
size == that.size &&
explain == that.explain &&
version == that.version &&
seqNoAndPrimaryTerm == that.seqNoAndPrimaryTerm &&
trackScores == that.trackScores &&
Objects.equals(sorts, that.sorts) &&
Objects.equals(highlightBuilder, that.highlightBuilder) &&
Objects.equals(storedFieldsContext, that.storedFieldsContext) &&
Objects.equals(docValueFields, that.docValueFields) &&
Objects.equals(fetchFields, that.fetchFields) &&
Objects.equals(scriptFields, that.scriptFields) &&
Objects.equals(fetchSourceContext, that.fetchSourceContext);
}
@Override
public boolean equals(Object obj) {
if (this == obj) return true;
if (obj == null || getClass() != obj.getClass()) return false;
if (super.equals(obj) == false) return false;
TopHitsAggregationBuilder other = (TopHitsAggregationBuilder) obj;
return Objects.equals(explain, other.explain)
&& Objects.equals(fetchSourceContext, other.fetchSourceContext)
&& Objects.equals(docValueFields, other.docValueFields)
&& Objects.equals(storedFieldsContext, other.storedFieldsContext)
&& Objects.equals(from, other.from)
&& Objects.equals(highlightBuilder, other.highlightBuilder)
&& Objects.equals(scriptFields, other.scriptFields)
&& Objects.equals(size, other.size)
&& Objects.equals(sorts, other.sorts)
&& Objects.equals(trackScores, other.trackScores)
&& Objects.equals(version, other.version)
&& Objects.equals(seqNoAndPrimaryTerm, other.seqNoAndPrimaryTerm);
public int hashCode() {
return Objects.hash(super.hashCode(), from, size, explain, version, seqNoAndPrimaryTerm, trackScores, sorts, highlightBuilder,
storedFieldsContext, docValueFields, fetchFields, scriptFields, fetchSourceContext);
}
@Override

9
server/src/main/java/org/elasticsearch/search/aggregations/metrics/TopHitsAggregatorFactory.java
View File

@ -26,6 +26,7 @@ import org.elasticsearch.search.aggregations.AggregatorFactory;
import org.elasticsearch.search.aggregations.CardinalityUpperBound;
import org.elasticsearch.search.fetch.StoredFieldsContext;
import org.elasticsearch.search.fetch.subphase.FetchDocValuesContext;
import org.elasticsearch.search.fetch.subphase.FetchFieldsContext;
import org.elasticsearch.search.fetch.subphase.FetchSourceContext;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
import org.elasticsearch.search.fetch.subphase.ScriptFieldsContext;
@ -51,6 +52,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
private final HighlightBuilder highlightBuilder;
private final StoredFieldsContext storedFieldsContext;
private final List<FieldAndFormat> docValueFields;
private final List<FieldAndFormat> fetchFields;
private final List<ScriptFieldsContext.ScriptField> scriptFields;
private final FetchSourceContext fetchSourceContext;
@ -65,6 +67,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
HighlightBuilder highlightBuilder,
StoredFieldsContext storedFieldsContext,
List<FieldAndFormat> docValueFields,
List<FieldAndFormat> fetchFields,
List<ScriptFieldsContext.ScriptField> scriptFields,
FetchSourceContext fetchSourceContext,
QueryShardContext queryShardContext,
@ -82,6 +85,7 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
this.highlightBuilder = highlightBuilder;
this.storedFieldsContext = storedFieldsContext;
this.docValueFields = docValueFields;
this.fetchFields = fetchFields;
this.scriptFields = scriptFields;
this.fetchSourceContext = fetchSourceContext;
}
@ -109,6 +113,11 @@ class TopHitsAggregatorFactory extends AggregatorFactory {
FetchDocValuesContext docValuesContext = FetchDocValuesContext.create(searchContext.mapperService(), docValueFields);
subSearchContext.docValuesContext(docValuesContext);
}
if (fetchFields != null) {
String indexName = searchContext.indexShard().shardId().getIndexName();
FetchFieldsContext fieldsContext = FetchFieldsContext.create(indexName, searchContext.mapperService(), fetchFields);
subSearchContext.fetchFieldsContext(fieldsContext);
}
for (ScriptFieldsContext.ScriptField field : scriptFields) {
subSearchContext.scriptFields().add(field);
}

10
server/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View File

@ -172,7 +172,7 @@ public class FetchPhase {
if (!context.hasScriptFields() && !context.hasFetchSourceContext()) {
context.fetchSourceContext(new FetchSourceContext(true));
}
boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null;
boolean loadSource = sourceRequired(context);
return new FieldsVisitor(loadSource);
} else if (storedFieldsContext.fetchFields() == false) {
// disable stored fields entirely
@ -202,7 +202,7 @@ public class FetchPhase {
}
}
}
boolean loadSource = context.sourceRequested() || context.fetchFieldsContext() != null;
boolean loadSource = sourceRequired(context);
if (storedToRequestedFields.isEmpty()) {
// empty list specified, default to disable _source if no explicit indication
return new FieldsVisitor(loadSource);
@ -212,6 +212,10 @@ public class FetchPhase {
}
}
private boolean sourceRequired(SearchContext context) {
return context.sourceRequested() || context.fetchFieldsContext() != null;
}
private int findRootDocumentIfNested(SearchContext context, LeafReaderContext subReaderContext, int subDocId) throws IOException {
if (context.mapperService().hasNested()) {
BitSet bits = context.bitsetFilterCache()
@ -295,7 +299,7 @@ public class FetchPhase {
// Also if highlighting is requested on nested documents we need to fetch the _source from the root document,
// otherwise highlighting will attempt to fetch the _source from the nested doc, which will fail,
// because the entire _source is only stored with the root document.
boolean needSource = context.sourceRequested() || context.highlight() != null;
boolean needSource = sourceRequired(context) || context.highlight() != null;
Uid rootId;
Map<String, Object> rootSourceAsMap = null;

19
server/src/test/java/org/elasticsearch/index/query/InnerHitBuilderTests.java
View File

@ -158,6 +158,8 @@ public class InnerHitBuilderTests extends ESTestCase {
}
innerHits.setDocValueFields(randomListStuff(16,
() -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)));
innerHits.setFetchFields(randomListStuff(16,
() -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null)));
// Random script fields deduped on their field name.
Map<String, SearchSourceBuilder.ScriptField> scriptFields = new HashMap<>();
for (SearchSourceBuilder.ScriptField field: randomListStuff(16, InnerHitBuilderTests::randomScript)) {
@ -204,6 +206,14 @@ public class InnerHitBuilderTests extends ESTestCase {
copy.addDocValueField(randomAlphaOfLengthBetween(1, 16));
}
});
modifiers.add(() -> {
if (randomBoolean()) {
copy.setFetchFields(randomValueOtherThan(copy.getFetchFields(),
() -> randomListStuff(16, () -> new FieldAndFormat(randomAlphaOfLengthBetween(1, 16), null))));
} else {
copy.addFetchField(randomAlphaOfLengthBetween(1, 16));
}
});
modifiers.add(() -> {
if (randomBoolean()) {
copy.setScriptFields(randomValueOtherThan(copy.getScriptFields(), () -> {
@ -292,4 +302,13 @@ public class InnerHitBuilderTests extends ESTestCase {
Arrays.asList(new FieldAndFormat("foo", null), new FieldAndFormat("@timestamp", "epoch_millis")),
innerHit.getDocValueFields());
}
public void testSetFetchFieldFormat() {
InnerHitBuilder innerHit = new InnerHitBuilder();
innerHit.addFetchField("foo");
innerHit.addFetchField("@timestamp", "epoch_millis");
assertEquals(
Arrays.asList(new FieldAndFormat("foo", null), new FieldAndFormat("@timestamp", "epoch_millis")),
innerHit.getFetchFields());
}
}

6
server/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsTests.java
View File

@ -86,6 +86,12 @@ public class TopHitsTests extends BaseAggregationTestCase<TopHitsAggregationBuil
factory.docValueField(randomAlphaOfLengthBetween(5, 50));
}
}
if (randomBoolean()) {
int fetchFieldsSize = randomInt(25);
for (int i = 0; i < fetchFieldsSize; i++) {
factory.fetchField(randomAlphaOfLengthBetween(5, 50));
}
}
if (randomBoolean()) {
int scriptFieldsSize = randomInt(25);
for (int i = 0; i < scriptFieldsSize; i++) {

3
x-pack/plugin/runtime-fields/qa/rest/build.gradle
View File

@ -32,7 +32,8 @@ yamlRestTest {
systemProperty 'tests.rest.blacklist',
[
/////// TO FIX ///////
'search/330_fetch_fields/*', // The whole API is not yet supported
'search/330_fetch_fields/*', // The 'fields' option is not yet supported
'search/110_field_collapsing/field collapsing, inner_hits, and fields', // Also fails because of the 'fields' option
'search.highlight/40_keyword_ignore/Plain Highligher should skip highlighting ignored keyword values', // The plain highlighter is incompatible with runtime fields. Worth fixing?
'search/115_multiple_field_collapsing/two levels fields collapsing', // Broken. Gotta fix.
'field_caps/30_filter/Field caps with index filter', // We don't support filtering field caps on runtime fields. What should we do?