honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Return the _source of inner hit nested as is without wrapping it into its full path context 

Due to a change happened via #26102 to make the nested source consistent
with or without source filtering, the _source of a nested inner hit was
always wrapped in the parent path. This turned out to be not ideal for
users relying on the nested source, as it would require additional parsing
on the client side. This change fixes this, the _source of nested inner hits
is now no longer wrapped by parent json objects, irregardless of whether
the _source is included as is or source filtering is used.

Internally source filtering and highlighting relies on the fact that the
_source of nested inner hits are accessible by its full field path, so
in order to now break this, the conversion of the _source into its binary
form is performed in FetchSourceSubPhase, after any potential source filtering
is performed to make sure the structure of _source of the nested inner hit
is consistent irregardless if source filtering is performed.

PR for #26944

Closes #26944
This commit is contained in:
Martijn van Groningen 2017-10-12 11:29:01 +02:00
parent 9a3a1cd1b7
commit 87c9b79b10
No known key found for this signature in database
GPG Key ID: AB236F4FCF2AF12A
6 changed files with 57 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/src/main/java/org/elasticsearch/search/fetch/FetchPhase.java
View File

@ -301,8 +301,6 @@ public class FetchPhase implements SearchPhase {
}
context.lookup().source().setSource(nestedSourceAsMap);
XContentType contentType = tuple.v1();
BytesReference nestedSource = contentBuilder(contentType).map(nestedSourceAsMap).bytes();
context.lookup().source().setSource(nestedSource);
context.lookup().source().setSourceContentType(contentType);
}
return new SearchHit(nestedTopDocId, uid.id(), documentMapper.typeText(), nestedIdentity, searchFields);

36
core/src/main/java/org/elasticsearch/search/fetch/subphase/FetchSourceSubPhase.java
View File

@ -20,13 +20,20 @@
package org.elasticsearch.search.fetch.subphase;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.fetch.FetchSubPhase;
import org.elasticsearch.search.internal.SearchContext;
import org.elasticsearch.search.lookup.SourceLookup;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.util.Map;
import static org.elasticsearch.common.xcontent.XContentFactory.contentBuilder;
public final class FetchSourceSubPhase implements FetchSubPhase {
@ -35,22 +42,27 @@ public final class FetchSourceSubPhase implements FetchSubPhase {
if (context.sourceRequested() == false) {
return;
}
final boolean nestedHit = hitContext.hit().getNestedIdentity() != null;
SourceLookup source = context.lookup().source();
FetchSourceContext fetchSourceContext = context.fetchSourceContext();
assert fetchSourceContext.fetchSource();
if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) {
hitContext.hit().sourceRef(source.internalSourceRef());
return;
if (nestedHit == false) {
if (fetchSourceContext.includes().length == 0 && fetchSourceContext.excludes().length == 0) {
hitContext.hit().sourceRef(source.internalSourceRef());
return;
}
if (source.internalSourceRef() == null) {
throw new IllegalArgumentException("unable to fetch fields from _source field: _source is disabled in the mappings " +
"for index [" + context.indexShard().shardId().getIndexName() + "]");
}
}
if (source.internalSourceRef() == null) {
throw new IllegalArgumentException("unable to fetch fields from _source field: _source is disabled in the mappings " +
"for index [" + context.indexShard().shardId().getIndexName() + "]");
Object value = source.filter(fetchSourceContext);
if (nestedHit) {
value = getNestedSource((Map<String, Object>) value, hitContext);
}
final Object value = source.filter(fetchSourceContext);
try {
final int initialCapacity = Math.min(1024, source.internalSourceRef().length());
final int initialCapacity = nestedHit ? 1024 : Math.min(1024, source.internalSourceRef().length());
BytesStreamOutput streamOutput = new BytesStreamOutput(initialCapacity);
XContentBuilder builder = new XContentBuilder(source.sourceContentType().xContent(), streamOutput);
builder.value(value);
@ -58,6 +70,12 @@ public final class FetchSourceSubPhase implements FetchSubPhase {
} catch (IOException e) {
throw new ElasticsearchException("Error filtering source", e);
}
}
private Map<String, Object> getNestedSource(Map<String, Object> sourceAsMap, HitContext hitContext) {
for (SearchHit.NestedIdentity o = hitContext.hit().getNestedIdentity(); o != null; o = o.getChild()) {
sourceAsMap = (Map<String, Object>) sourceAsMap.get(o.getField().string());
}
return sourceAsMap;
}
}

26
core/src/test/java/org/elasticsearch/search/aggregations/metrics/TopHitsIT.java
View File

@ -729,7 +729,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(searchHits.getTotalHits(), equalTo(1L));
assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(0));
assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(1));
assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(1));
bucket = terms.getBucketByKey("b");
assertThat(bucket.getDocCount(), equalTo(2L));
@ -738,10 +738,10 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(searchHits.getTotalHits(), equalTo(2L));
assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(2));
assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(2));
assertThat(searchHits.getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(searchHits.getAt(1).getNestedIdentity().getOffset(), equalTo(0));
assertThat(extractValue("comments.date", searchHits.getAt(1).getSourceAsMap()), equalTo(3));
assertThat(extractValue("date", searchHits.getAt(1).getSourceAsMap()), equalTo(3));
bucket = terms.getBucketByKey("c");
assertThat(bucket.getDocCount(), equalTo(1L));
@ -750,7 +750,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(searchHits.getTotalHits(), equalTo(1L));
assertThat(searchHits.getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(searchHits.getAt(0).getNestedIdentity().getOffset(), equalTo(1));
assertThat(extractValue("comments.date", searchHits.getAt(0).getSourceAsMap()), equalTo(4));
assertThat(extractValue("date", searchHits.getAt(0).getSourceAsMap()), equalTo(4));
}
public void testTopHitsInSecondLayerNested() throws Exception {
@ -803,49 +803,49 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(topReviewers.getHits().getHits().length, equalTo(7));
assertThat(topReviewers.getHits().getAt(0).getId(), equalTo("1"));
assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(0).getSourceAsMap()), equalTo("user a"));
assertThat(extractValue("name", topReviewers.getHits().getAt(0).getSourceAsMap()), equalTo("user a"));
assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0));
assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getOffset(), equalTo(0));
assertThat(topReviewers.getHits().getAt(1).getId(), equalTo("1"));
assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(1).getSourceAsMap()), equalTo("user b"));
assertThat(extractValue("name", topReviewers.getHits().getAt(1).getSourceAsMap()), equalTo("user b"));
assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getOffset(), equalTo(0));
assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
assertThat(topReviewers.getHits().getAt(1).getNestedIdentity().getChild().getOffset(), equalTo(1));
assertThat(topReviewers.getHits().getAt(2).getId(), equalTo("1"));
assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(2).getSourceAsMap()), equalTo("user c"));
assertThat(extractValue("name", topReviewers.getHits().getAt(2).getSourceAsMap()), equalTo("user c"));
assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getOffset(), equalTo(0));
assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
assertThat(topReviewers.getHits().getAt(2).getNestedIdentity().getChild().getOffset(), equalTo(2));
assertThat(topReviewers.getHits().getAt(3).getId(), equalTo("1"));
assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(3).getSourceAsMap()), equalTo("user c"));
assertThat(extractValue("name", topReviewers.getHits().getAt(3).getSourceAsMap()), equalTo("user c"));
assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getOffset(), equalTo(1));
assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
assertThat(topReviewers.getHits().getAt(3).getNestedIdentity().getChild().getOffset(), equalTo(0));
assertThat(topReviewers.getHits().getAt(4).getId(), equalTo("1"));
assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(4).getSourceAsMap()), equalTo("user d"));
assertThat(extractValue("name", topReviewers.getHits().getAt(4).getSourceAsMap()), equalTo("user d"));
assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getOffset(), equalTo(1));
assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
assertThat(topReviewers.getHits().getAt(4).getNestedIdentity().getChild().getOffset(), equalTo(1));
assertThat(topReviewers.getHits().getAt(5).getId(), equalTo("1"));
assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(5).getSourceAsMap()), equalTo("user e"));
assertThat(extractValue("name", topReviewers.getHits().getAt(5).getSourceAsMap()), equalTo("user e"));
assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getOffset(), equalTo(1));
assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
assertThat(topReviewers.getHits().getAt(5).getNestedIdentity().getChild().getOffset(), equalTo(2));
assertThat(topReviewers.getHits().getAt(6).getId(), equalTo("2"));
assertThat(extractValue("comments.reviewers.name", topReviewers.getHits().getAt(6).getSourceAsMap()), equalTo("user f"));
assertThat(extractValue("name", topReviewers.getHits().getAt(6).getSourceAsMap()), equalTo("user f"));
assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getOffset(), equalTo(0));
assertThat(topReviewers.getHits().getAt(0).getNestedIdentity().getChild().getField().string(), equalTo("reviewers"));
@ -901,7 +901,7 @@ public class TopHitsIT extends ESIntegTestCase {
assertThat(field.getValue().toString(), equalTo("5"));
assertThat(searchHit.getSourceAsMap().size(), equalTo(1));
assertThat(extractValue("comments.message", searchHit.getSourceAsMap()), equalTo("some comment"));
assertThat(extractValue("message", searchHit.getSourceAsMap()), equalTo("some comment"));
}
public void testTopHitsInNested() throws Exception {
@ -934,7 +934,7 @@ public class TopHitsIT extends ESIntegTestCase {
for (int j = 0; j < 3; j++) {
assertThat(searchHits.getAt(j).getNestedIdentity().getField().string(), equalTo("comments"));
assertThat(searchHits.getAt(j).getNestedIdentity().getOffset(), equalTo(0));
assertThat(extractValue("comments.id", searchHits.getAt(j).getSourceAsMap()), equalTo(0));
assertThat(extractValue("id", searchHits.getAt(j).getSourceAsMap()), equalTo(0));
HighlightField highlightField = searchHits.getAt(j).getHighlightFields().get("comments.message");
assertThat(highlightField.getFragments().length, equalTo(1));

18
core/src/test/java/org/elasticsearch/search/fetch/subphase/InnerHitsIT.java
View File

@ -596,9 +596,9 @@ public class InnerHitsIT extends ESIntegTestCase {
client().prepareIndex("index1", "message", "1").setSource(jsonBuilder().startObject()
.field("message", "quick brown fox")
.startArray("comments")
.startObject().field("message", "fox eat quick").endObject()
.startObject().field("message", "fox ate rabbit x y z").endObject()
.startObject().field("message", "rabbit got away").endObject()
.startObject().field("message", "fox eat quick").field("x", "y").endObject()
.startObject().field("message", "fox ate rabbit x y z").field("x", "y").endObject()
.startObject().field("message", "rabbit got away").field("x", "y").endObject()
.endArray()
.endObject()).get();
refresh();
@ -614,9 +614,11 @@ public class InnerHitsIT extends ESIntegTestCase {
assertHitCount(response, 1);
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L));
assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap()),
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"),
equalTo("fox eat quick"));
assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap()),
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().size(), equalTo(1));
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().get("message"),
equalTo("fox ate rabbit x y z"));
response = client().prepareSearch()
@ -627,9 +629,11 @@ public class InnerHitsIT extends ESIntegTestCase {
assertHitCount(response, 1);
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getTotalHits(), equalTo(2L));
assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap()),
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().get("message"),
equalTo("fox eat quick"));
assertThat(extractValue("comments.message", response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap()),
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(0).getSourceAsMap().size(), equalTo(2));
assertThat(response.getHits().getAt(0).getInnerHits().get("comments").getAt(1).getSourceAsMap().get("message"),
equalTo("fox ate rabbit x y z"));
}

6
docs/reference/aggregations/metrics/tophits-aggregation.asciidoc
View File

@ -329,10 +329,8 @@ Top hits response snippet with a nested hit, which resides in the first slot of
},
"_score": 0.2876821,
"_source": {
"comments": {
"comment": "This car could have better brakes", <3>
"username": "baddriver007"
}
"comment": "This car could have better brakes", <3>
"username": "baddriver007"
}
}
]

14
docs/reference/search/request/inner-hits.asciidoc
View File

@ -158,10 +158,8 @@ An example of a response snippet that could be generated from the above search r
},
"_score": 1.0,
"_source": {
"comments" : {
"author": "nik9000",
"number": 2
}
"author": "nik9000",
"number": 2
}
}
]
@ -406,12 +404,8 @@ Which would look like:
},
"_score": 0.6931472,
"_source": {
"comments": {
"votes": {
"value": 1,
"voter": "kimchy"
}
}
"value": 1,
"voter": "kimchy"
}
}
]