diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index ab628da46aa..e620fa63d53 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -180,4 +181,16 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { public static DiscountedCumulativeGainAt fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(SIZE_FIELD.getPreferredName(), this.position); + builder.field(NORMALIZE_FIELD.getPreferredName(), this.normalize); + if (unknownDocRating != null) { + builder.field(UNKNOWN_DOC_RATING_FIELD.getPreferredName(), this.unknownDocRating); + } + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index f97ba7c7975..ba571c576ef 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.SearchHit; @@ -150,4 +151,12 @@ public class PrecisionAtN extends RankedListQualityMetric { } } + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(NAME); + builder.field(SIZE_FIELD.getPreferredName(), this.n); + builder.endObject(); + return builder; + } + } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java index b316ab8d1d2..92445bd4697 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java @@ -19,12 +19,14 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.search.builder.SearchSourceBuilder; @@ -38,7 +40,7 @@ import java.util.List; * * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. * */ -public class QuerySpec implements Writeable { +public class QuerySpec extends ToXContentToBytes implements Writeable { private String specId; private SearchSourceBuilder testRequest; @@ -65,12 +67,12 @@ public class QuerySpec implements Writeable { this.specId = in.readString(); testRequest = new SearchSourceBuilder(in); int indicesSize = in.readInt(); - indices = new ArrayList(indicesSize); + indices = new ArrayList<>(indicesSize); for (int i = 0; i < indicesSize; i++) { this.indices.add(in.readString()); } int typesSize = in.readInt(); - types = new ArrayList(typesSize); + types = new ArrayList<>(typesSize); for (int i = 0; i < typesSize; i++) { this.types.add(in.readString()); } @@ -189,4 +191,18 @@ public class QuerySpec implements Writeable { public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException { return PARSER.parse(parser, context); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(ID_FIELD.getPreferredName(), this.specId); + builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); + builder.startArray(RATINGS_FIELD.getPreferredName()); + for (RatedDocument doc : this.ratedDocs) { + doc.toXContent(builder, params); + } + builder.endArray(); + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 8f3f27a792e..7f88104b4eb 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -19,36 +19,42 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; /** - * This class defines a qa task including query intent and query spec. + * This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric. * * Each QA run is based on a set of queries to send to the index and multiple QA specifications that define how to translate the query - * intents into elastic search queries. In addition it contains the quality metrics to compute. + * intents into elastic search queries. * */ -public class RankEvalSpec implements Writeable { +public class RankEvalSpec extends ToXContentToBytes implements Writeable { /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ private Collection specifications = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric eval; /** a unique id for the whole QA task */ - private String taskId; + private String specId; public RankEvalSpec() { // TODO think if no args ctor is okay } - public RankEvalSpec(String taskId, Collection specs, RankedListQualityMetric metric) { - this.taskId = taskId; + public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { + this.specId = specId; this.specifications = specs; this.eval = metric; } @@ -60,7 +66,7 @@ public class RankEvalSpec implements Writeable { specifications.add(new QuerySpec(in)); } eval = in.readNamedWriteable(RankedListQualityMetric.class); - taskId = in.readString(); + specId = in.readString(); } @Override @@ -70,7 +76,7 @@ public class RankEvalSpec implements Writeable { spec.writeTo(out); } out.writeNamedWriteable(eval); - out.writeString(taskId); + out.writeString(specId); } public void setEval(RankedListQualityMetric eval) { @@ -78,11 +84,11 @@ public class RankEvalSpec implements Writeable { } public void setTaskId(String taskId) { - this.taskId = taskId; + this.specId = taskId; } public String getTaskId() { - return this.taskId; + return this.specId; } /** Returns the precision at n configuration (containing level of n to consider).*/ @@ -105,4 +111,45 @@ public class RankEvalSpec implements Writeable { this.specifications = specifications; } + private static final ParseField SPECID_FIELD = new ParseField("spec_id"); + private static final ParseField METRIC_FIELD = new ParseField("metric"); + private static final ParseField REQUESTS_FIELD = new ParseField("requests"); + private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); + + static { + PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD); + PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { + try { + return RankedListQualityMetric.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , METRIC_FIELD); + PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { + try { + return QuerySpec.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , REQUESTS_FIELD); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(SPECID_FIELD.getPreferredName(), this.specId); + builder.startArray(REQUESTS_FIELD.getPreferredName()); + for (QuerySpec spec : this.specifications) { + spec.toXContent(builder, params); + } + builder.endArray(); + builder.field(METRIC_FIELD.getPreferredName(), this.eval); + builder.endObject(); + return builder; + } + + public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.parse(parser, context); + } + } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 86c7ebf38be..e423ab3533c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -19,9 +19,11 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.search.SearchHit; @@ -36,7 +38,7 @@ import java.util.List; * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public abstract class RankedListQualityMetric implements NamedWriteable { +public abstract class RankedListQualityMetric extends ToXContentToBytes implements NamedWriteable { /** * Returns a single metric representing the ranking quality of a set of returned documents @@ -79,4 +81,7 @@ public abstract class RankedListQualityMetric implements NamedWriteable { double combine(Collection partialResults) { return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size(); } + + @Override + public abstract XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 5f23c662ea1..dd4e710859b 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -24,6 +24,7 @@ import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.index.rankeval.PrecisionAtN.RatingMapping; @@ -142,4 +143,14 @@ public class ReciprocalRank extends RankedListQualityMetric { public static ReciprocalRank fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) { return PARSER.apply(parser, matcher); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.startObject(NAME); + builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); + builder.endObject(); + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 8316dc49db3..8acc7b34a51 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -20,13 +20,10 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; @@ -192,34 +189,13 @@ public class RestRankEvalAction extends BaseRestHandler { client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener(channel)); } - private static final ParseField SPECID_FIELD = new ParseField("spec_id"); - private static final ParseField METRIC_FIELD = new ParseField("metric"); - private static final ParseField REQUESTS_FIELD = new ParseField("requests"); - private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); - static { - PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD); - PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { - try { - return RankedListQualityMetric.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } - } , METRIC_FIELD); - PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { - try { - return QuerySpec.fromXContent(p, c); - } catch (IOException ex) { - throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); - } - } , REQUESTS_FIELD); - } public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context) throws IOException { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); - RankEvalSpec spec = PARSER.parse(context.parser(), context); + RankEvalSpec spec = RankEvalSpec.parse(context.parser(), context); for (QuerySpec specification : spec.getSpecifications()) { specification.setIndices(indices); specification.setTypes(types);