honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add fromXContent method to SearchResponse (#24720) 

SearchResponse#fromXContent allows to parse a search response, including search hits, aggregations, suggestions and profile results. Only the aggs that we can parse today are supported (which means all of them but a couple that are left to support). SearchResponseTests reuses the existing test infra to randomize aggregations, suggestions and profile response.

Relates to #23331
This commit is contained in:
Luca Cavanna 2017-05-17 20:28:33 +02:00 committed by GitHub
parent 9fc9db26fd
commit da669f0554
10 changed files with 318 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
core/src/main/java/org/elasticsearch/action/search/SearchResponse.java
View File

@ -21,30 +21,45 @@ package org.elasticsearch.action.search;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.StatusToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.action.RestActions;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.profile.ProfileShardResult;
import org.elasticsearch.search.profile.SearchProfileShardResults;
import org.elasticsearch.search.suggest.Suggest;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import static org.elasticsearch.action.search.ShardSearchFailure.readShardSearchFailure;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownField;
import static org.elasticsearch.common.xcontent.XContentParserUtils.throwUnknownToken;
/**
* A response of a search request.
*/
public class SearchResponse extends ActionResponse implements StatusToXContentObject {
private static final ParseField SCROLL_ID = new ParseField("_scroll_id");
private static final ParseField TOOK = new ParseField("took");
private static final ParseField TIMED_OUT = new ParseField("timed_out");
private static final ParseField TERMINATED_EARLY = new ParseField("terminated_early");
private static final ParseField NUM_REDUCE_PHASES = new ParseField("num_reduce_phases");
private SearchResponseSections internalResponse;
private String scrollId;
@ -175,7 +190,8 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
*
* @return The profile results or an empty map
*/
@Nullable public Map<String, ProfileShardResult> getProfileResults() {
@Nullable
public Map<String, ProfileShardResult> getProfileResults() {
return internalResponse.profile();
}
@ -189,15 +205,15 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
public XContentBuilder innerToXContent(XContentBuilder builder, Params params) throws IOException {
if (scrollId != null) {
builder.field("_scroll_id", scrollId);
builder.field(SCROLL_ID.getPreferredName(), scrollId);
}
builder.field("took", tookInMillis);
builder.field("timed_out", isTimedOut());
builder.field(TOOK.getPreferredName(), tookInMillis);
builder.field(TIMED_OUT.getPreferredName(), isTimedOut());
if (isTerminatedEarly() != null) {
builder.field("terminated_early", isTerminatedEarly());
builder.field(TERMINATED_EARLY.getPreferredName(), isTerminatedEarly());
}
if (getNumReducePhases() != 1) {
builder.field("num_reduce_phases", getNumReducePhases());
builder.field(NUM_REDUCE_PHASES.getPreferredName(), getNumReducePhases());
}
RestActions.buildBroadcastShardsHeader(builder, params, getTotalShards(), getSuccessfulShards(), getFailedShards(),
getShardFailures());
@ -205,6 +221,85 @@ public class SearchResponse extends ActionResponse implements StatusToXContentOb
return builder;
}
public static SearchResponse fromXContent(XContentParser parser) throws IOException {
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
XContentParser.Token token;
String currentFieldName = null;
SearchHits hits = null;
Aggregations aggs = null;
Suggest suggest = null;
SearchProfileShardResults profile = null;
boolean timedOut = false;
Boolean terminatedEarly = null;
int numReducePhases = 1;
long tookInMillis = -1;
int successfulShards = -1;
int totalShards = -1;
String scrollId = null;
List<ShardSearchFailure> failures = new ArrayList<>();
while((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if (SCROLL_ID.match(currentFieldName)) {
scrollId = parser.text();
} else if (TOOK.match(currentFieldName)) {
tookInMillis = parser.longValue();
} else if (TIMED_OUT.match(currentFieldName)) {
timedOut = parser.booleanValue();
} else if (TERMINATED_EARLY.match(currentFieldName)) {
terminatedEarly = parser.booleanValue();
} else if (NUM_REDUCE_PHASES.match(currentFieldName)) {
numReducePhases = parser.intValue();
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (SearchHits.Fields.HITS.equals(currentFieldName)) {
hits = SearchHits.fromXContent(parser);
} else if (Aggregations.AGGREGATIONS_FIELD.equals(currentFieldName)) {
aggs = Aggregations.fromXContent(parser);
} else if (Suggest.NAME.equals(currentFieldName)) {
suggest = Suggest.fromXContent(parser);
} else if (SearchProfileShardResults.PROFILE_FIELD.equals(currentFieldName)) {
profile = SearchProfileShardResults.fromXContent(parser);
} else if (RestActions._SHARDS_FIELD.match(currentFieldName)) {
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token.isValue()) {
if (RestActions.FAILED_FIELD.match(currentFieldName)) {
parser.intValue(); // we don't need it but need to consume it
} else if (RestActions.SUCCESSFUL_FIELD.match(currentFieldName)) {
successfulShards = parser.intValue();
} else if (RestActions.TOTAL_FIELD.match(currentFieldName)) {
totalShards = parser.intValue();
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (RestActions.FAILURES_FIELD.match(currentFieldName)) {
while((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
failures.add(ShardSearchFailure.fromXContent(parser));
}
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
} else {
throwUnknownToken(token, parser.getTokenLocation());
}
}
} else {
throwUnknownField(currentFieldName, parser.getTokenLocation());
}
}
}
SearchResponseSections searchResponseSections = new SearchResponseSections(hits, aggs, suggest, timedOut, terminatedEarly,
profile, numReducePhases);
return new SearchResponse(searchResponseSections, scrollId, totalShards, successfulShards, tookInMillis,
failures.toArray(new ShardSearchFailure[failures.size()]));
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);

17
core/src/main/java/org/elasticsearch/rest/action/RestActions.java
View File

@ -25,6 +25,7 @@ import org.elasticsearch.action.ShardOperationFailedException;
import org.elasticsearch.action.support.broadcast.BroadcastResponse;
import org.elasticsearch.action.support.nodes.BaseNodeResponse;
import org.elasticsearch.action.support.nodes.BaseNodesResponse;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.lucene.uid.Versions;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContent.Params;
@ -46,6 +47,12 @@ import java.util.List;
public class RestActions {
public static final ParseField _SHARDS_FIELD = new ParseField("_shards");
public static final ParseField TOTAL_FIELD = new ParseField("total");
public static final ParseField SUCCESSFUL_FIELD = new ParseField("successful");
public static final ParseField FAILED_FIELD = new ParseField("failed");
public static final ParseField FAILURES_FIELD = new ParseField("failures");
public static long parseVersion(RestRequest request) {
if (request.hasParam("version")) {
return request.paramAsLong("version", Versions.MATCH_ANY);
@ -71,12 +78,12 @@ public class RestActions {
public static void buildBroadcastShardsHeader(XContentBuilder builder, Params params,
int total, int successful, int failed,
ShardOperationFailedException[] shardFailures) throws IOException {
builder.startObject("_shards");
builder.field("total", total);
builder.field("successful", successful);
builder.field("failed", failed);
builder.startObject(_SHARDS_FIELD.getPreferredName());
builder.field(TOTAL_FIELD.getPreferredName(), total);
builder.field(SUCCESSFUL_FIELD.getPreferredName(), successful);
builder.field(FAILED_FIELD.getPreferredName(), failed);
if (shardFailures != null && shardFailures.length > 0) {
builder.startArray("failures");
builder.startArray(FAILURES_FIELD.getPreferredName());
final boolean group = params.paramAsBoolean("group_shard_failures", true); // we group by default
for (ShardOperationFailedException shardFailure : group ? ExceptionsHelper.groupBy(shardFailures) : shardFailures) {
builder.startObject();

8
core/src/main/java/org/elasticsearch/search/SearchHits.java
View File

@ -105,10 +105,10 @@ public final class SearchHits implements Streamable, ToXContent, Iterable<Search
return this.hits;
}
static final class Fields {
static final String HITS = "hits";
static final String TOTAL = "total";
static final String MAX_SCORE = "max_score";
public static final class Fields {
public static final String HITS = "hits";
public static final String TOTAL = "total";
public static final String MAX_SCORE = "max_score";
}
@Override

2
core/src/main/java/org/elasticsearch/search/suggest/Suggest.java
View File

@ -60,7 +60,7 @@ import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpect
*/
public class Suggest implements Iterable<Suggest.Suggestion<? extends Entry<? extends Option>>>, Streamable, ToXContent {
static final String NAME = "suggest";
public static final String NAME = "suggest";
public static final Comparator<Option> COMPARATOR = (first, second) -> {
int cmp = Float.compare(second.getScore(), first.getScore());

180
core/src/test/java/org/elasticsearch/action/search/SearchResponseTests.java Normal file
View File

@ -0,0 +1,180 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.search;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.rest.action.search.RestSearchAction;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.SearchHitsTests;
import org.elasticsearch.search.aggregations.AggregationsTests;
import org.elasticsearch.search.aggregations.InternalAggregations;
import org.elasticsearch.search.internal.InternalSearchResponse;
import org.elasticsearch.search.profile.SearchProfileShardResults;
import org.elasticsearch.search.profile.SearchProfileShardResultsTests;
import org.elasticsearch.search.suggest.Suggest;
import org.elasticsearch.search.suggest.SuggestTests;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.InternalAggregationTestCase;
import org.junit.After;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static java.util.Collections.singletonMap;
import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent;
public class SearchResponseTests extends ESTestCase {
private static final NamedXContentRegistry xContentRegistry;
static {
List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>(InternalAggregationTestCase.getDefaultNamedXContents());
namedXContents.addAll(SuggestTests.getDefaultNamedXContents());
xContentRegistry = new NamedXContentRegistry(namedXContents);
}
private AggregationsTests aggregationsTests = new AggregationsTests();
@Before
public void init() throws Exception {
aggregationsTests.init();
}
@After
public void cleanUp() throws Exception {
aggregationsTests.cleanUp();
}
@Override
protected NamedXContentRegistry xContentRegistry() {
return xContentRegistry;
}
private SearchResponse createTestItem(ShardSearchFailure... shardSearchFailures) {
SearchHits hits = SearchHitsTests.createTestItem();
boolean timedOut = randomBoolean();
Boolean terminatedEarly = randomBoolean() ? null : randomBoolean();
int numReducePhases = randomIntBetween(1, 10);
long tookInMillis = randomNonNegativeLong();
int successfulShards = randomInt();
int totalShards = randomInt();
InternalAggregations aggregations = aggregationsTests.createTestInstance();
Suggest suggest = SuggestTests.createTestItem();
SearchProfileShardResults profileShardResults = SearchProfileShardResultsTests.createTestItem();
InternalSearchResponse internalSearchResponse = new InternalSearchResponse(hits, aggregations, suggest, profileShardResults,
timedOut, terminatedEarly, numReducePhases);
return new SearchResponse(internalSearchResponse, null, totalShards, successfulShards, tookInMillis, shardSearchFailures);
}
public void testFromXContent() throws IOException {
// the "_shard/total/failures" section makes if impossible to directly compare xContent, so we omit it here
SearchResponse response = createTestItem();
XContentType xcontentType = randomFrom(XContentType.values());
boolean humanReadable = randomBoolean();
final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
BytesReference originalBytes = toShuffledXContent(response, xcontentType, params, humanReadable);
try (XContentParser parser = createParser(xcontentType.xContent(), originalBytes)) {
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
SearchResponse parsed = SearchResponse.fromXContent(parser);
assertToXContentEquivalent(originalBytes, XContentHelper.toXContent(parsed, xcontentType, params, humanReadable), xcontentType);
assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
assertNull(parser.nextToken());
}
}
/**
* The "_shard/total/failures" section makes if impossible to directly compare xContent, because
* the failures in the parsed SearchResponse are wrapped in an extra ElasticSearchException on the client side.
* Because of this, in this special test case we compare the "top level" fields for equality
* and the subsections xContent equivalence independently
*/
public void testFromXContentWithFailures() throws IOException {
int numFailures = randomIntBetween(1, 5);
ShardSearchFailure[] failures = new ShardSearchFailure[numFailures];
for (int i = 0; i < failures.length; i++) {
failures[i] = ShardSearchFailureTests.createTestItem();
}
SearchResponse response = createTestItem(failures);
XContentType xcontentType = randomFrom(XContentType.values());
final ToXContent.Params params = new ToXContent.MapParams(singletonMap(RestSearchAction.TYPED_KEYS_PARAM, "true"));
BytesReference originalBytes = toShuffledXContent(response, xcontentType, params, randomBoolean());
try (XContentParser parser = createParser(xcontentType.xContent(), originalBytes)) {
ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.nextToken(), parser::getTokenLocation);
SearchResponse parsed = SearchResponse.fromXContent(parser);
for (int i = 0; i < parsed.getShardFailures().length; i++) {
ShardSearchFailure parsedFailure = parsed.getShardFailures()[i];
ShardSearchFailure originalFailure = failures[i];
assertEquals(originalFailure.index(), parsedFailure.index());
assertEquals(originalFailure.shard().getNodeId(), parsedFailure.shard().getNodeId());
assertEquals(originalFailure.shardId(), parsedFailure.shardId());
String originalMsg = originalFailure.getCause().getMessage();
assertEquals(parsedFailure.getCause().getMessage(), "Elasticsearch exception [type=parsing_exception, reason=" +
originalMsg + "]");
String nestedMsg = originalFailure.getCause().getCause().getMessage();
assertEquals(parsedFailure.getCause().getCause().getMessage(),
"Elasticsearch exception [type=illegal_argument_exception, reason=" + nestedMsg + "]");
}
assertEquals(XContentParser.Token.END_OBJECT, parser.currentToken());
assertNull(parser.nextToken());
}
}
public void testToXContent() {
SearchHit hit = new SearchHit(1, "id1", new Text("type"), Collections.emptyMap());
hit.score(2.0f);
SearchHit[] hits = new SearchHit[] { hit };
SearchResponse response = new SearchResponse(
new InternalSearchResponse(new SearchHits(hits, 100, 1.5f), null, null, null, false, null, 1), null, 0, 0, 0,
new ShardSearchFailure[0]);
StringBuilder expectedString = new StringBuilder();
expectedString.append("{");
{
expectedString.append("\"took\":0,");
expectedString.append("\"timed_out\":false,");
expectedString.append("\"_shards\":");
{
expectedString.append("{\"total\":0,");
expectedString.append("\"successful\":0,");
expectedString.append("\"failed\":0},");
}
expectedString.append("\"hits\":");
{
expectedString.append("{\"total\":100,");
expectedString.append("\"max_score\":1.5,");
expectedString.append("\"hits\":[{\"_type\":\"type\",\"_id\":\"id1\",\"_score\":2.0}]}");
}
}
expectedString.append("}");
assertEquals(expectedString.toString(), Strings.toString(response));
}
}

3
core/src/test/java/org/elasticsearch/action/search/ShardSearchFailureTests.java
View File

@ -22,6 +22,7 @@ package org.elasticsearch.action.search;
import org.elasticsearch.action.OriginalIndices;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.Index;
@ -50,7 +51,7 @@ public class ShardSearchFailureTests extends ESTestCase {
ShardSearchFailure response = createTestItem();
XContentType xContentType = randomFrom(XContentType.values());
boolean humanReadable = randomBoolean();
BytesReference originalBytes = toXContent(response, xContentType, humanReadable);
BytesReference originalBytes = toShuffledXContent(response, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
ShardSearchFailure parsed;
try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {

2
core/src/test/java/org/elasticsearch/search/aggregations/AggregationsTests.java
View File

@ -178,7 +178,7 @@ public class AggregationsTests extends ESTestCase {
}
}
private static InternalAggregations createTestInstance() {
public final InternalAggregations createTestInstance() {
return createTestInstance(1, 0, 5);
}

2
core/src/test/java/org/elasticsearch/search/profile/SearchProfileShardResultsTests.java
View File

@ -62,7 +62,7 @@ public class SearchProfileShardResultsTests extends ESTestCase {
XContentType xContentType = randomFrom(XContentType.values());
boolean humanReadable = randomBoolean();
BytesReference originalBytes = toShuffledXContent(shardResult, xContentType, ToXContent.EMPTY_PARAMS, humanReadable);
SearchProfileShardResults parsed = null;
SearchProfileShardResults parsed;
try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) {
ensureExpectedToken(parser.nextToken(), XContentParser.Token.START_OBJECT, parser::getTokenLocation);
ensureFieldName(parser, parser.nextToken(), SearchProfileShardResults.PROFILE_FIELD);

17
core/src/test/java/org/elasticsearch/search/suggest/SuggestTests.java
View File

@ -49,15 +49,26 @@ import static org.hamcrest.Matchers.equalTo;
public class SuggestTests extends ESTestCase {
static NamedXContentRegistry getSuggestersRegistry() {
List<NamedXContentRegistry.Entry> namedXContents = new ArrayList<>();
private static final NamedXContentRegistry xContentRegistry;
private static final List<NamedXContentRegistry.Entry> namedXContents;
static {
namedXContents = new ArrayList<>();
namedXContents.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField("term"),
(parser, context) -> TermSuggestion.fromXContent(parser, (String)context)));
namedXContents.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField("phrase"),
(parser, context) -> PhraseSuggestion.fromXContent(parser, (String)context)));
namedXContents.add(new NamedXContentRegistry.Entry(Suggest.Suggestion.class, new ParseField("completion"),
(parser, context) -> CompletionSuggestion.fromXContent(parser, (String)context)));
return new NamedXContentRegistry(namedXContents);
xContentRegistry = new NamedXContentRegistry(namedXContents);
}
public static List<NamedXContentRegistry.Entry> getDefaultNamedXContents() {
return namedXContents;
}
static NamedXContentRegistry getSuggestersRegistry() {
return xContentRegistry;
}
@Override

4
core/src/test/java/org/elasticsearch/search/suggest/SuggestionTests.java
View File

@ -68,8 +68,8 @@ public class SuggestionTests extends ESTestCase {
String name = randomAlphaOfLengthBetween(5, 10);
// note: size will not be rendered via "toXContent", only passed on internally on transport layer
int size = randomInt();
Supplier<Entry> entrySupplier = null;
Suggestion suggestion = null;
Supplier<Entry> entrySupplier;
Suggestion suggestion;
if (type == TermSuggestion.class) {
suggestion = new TermSuggestion(name, size, randomFrom(SortBy.values()));
entrySupplier = () -> SuggestionEntryTests.createTestItem(TermSuggestion.Entry.class);