Merge pull request #19283 from cbuescher/rank-add-restparsing

Query evaluation: Adding rest layer parsing and response rendering

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java

@@ -21,8 +21,8 @@ package org.elasticsearch.index.rankeval;
 
 import java.util.Collection;
 
-/** Returned for each search intent and search specification combination. Summarises the document ids found that were not
+/** Returned for each search specification. Summarizes the measured quality metric for this search request
- * annotated and the average precision of result sets in each particular combination based on the annotations given. 
+ * and adds the document ids found that were in the search result but not annotated in the original request.
  * */
 public class EvalQueryQuality {
     private double qualityLevel;

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java

@@ -1,28 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.rankeval;
-
-import org.elasticsearch.common.io.stream.NamedWriteable;
-import org.elasticsearch.search.SearchHit;
-
-public interface Evaluator extends NamedWriteable {
-
-    Object evaluate(SearchHit[] hits, RatedQuery intent);
-}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java

@@ -19,15 +19,18 @@
 
 package org.elasticsearch.index.rankeval;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ConstructingObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.SearchHit;
 
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.Map;
+import java.util.List;
-import java.util.Map.Entry;
 
 import javax.naming.directory.SearchResult;
 
@@ -36,7 +39,7 @@ import javax.naming.directory.SearchResult;
  *
  * Documents of unkonwn quality are ignored in the precision at n computation and returned by document id.
  * */
-public class PrecisionAtN implements RankedListQualityMetric {
+public class PrecisionAtN extends RankedListQualityMetric {
 
     /** Number of results to check against a given set of relevant results. */
     private int n;
@@ -78,24 +81,31 @@ public class PrecisionAtN implements RankedListQualityMetric {
         return n;
     }
 
+    private static final ParseField SIZE_FIELD = new ParseField("size");
+    private static final ConstructingObjectParser<PrecisionAtN, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
+            "precision_at", a -> new PrecisionAtN((Integer) a[0]));
+
+    static {
+        PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD);
+    }
+
+    public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) {
+        return PARSER.apply(parser, matcher);
+    }
+
     /** Compute precisionAtN based on provided relevant document IDs.
      * @return precision at n for above {@link SearchResult} list.
      **/
     @Override
-    public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent) {
+    public EvalQueryQuality evaluate(SearchHit[] hits, List<RatedDocument> ratedDocs) {
-        Map<String, Integer> ratedDocIds = intent.getRatedDocuments();
 
         Collection<String> relevantDocIds = new ArrayList<>();
-        for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
-            if (Rating.RELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
-                relevantDocIds.add(entry.getKey());
-            }
-        }
-        
         Collection<String> irrelevantDocIds = new ArrayList<>();
-        for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
+        for (RatedDocument doc : ratedDocs) {
-            if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
+            if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
-                irrelevantDocIds.add(entry.getKey());
+                relevantDocIds.add(doc.getDocID());
+            } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
+                irrelevantDocIds.add(doc.getDocID());
             }
         }
 
@@ -119,7 +129,7 @@ public class PrecisionAtN implements RankedListQualityMetric {
     }
 
     public enum Rating {
-        RELEVANT, IRRELEVANT;
+        IRRELEVANT, RELEVANT;
     }
 
     /**
@@ -128,13 +138,13 @@ public class PrecisionAtN implements RankedListQualityMetric {
     public static class RatingMapping {
         public static Integer mapFrom(Rating rating) {
             if (Rating.RELEVANT.equals(rating)) {
-                return 0;
-            }
                 return 1;
             }
+            return 0;
+        }
 
         public static Rating mapTo(Integer rating) {
-            if (rating == 0) {
+            if (rating == 1) {
                 return Rating.RELEVANT;
             }
             return Rating.IRRELEVANT;

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java

@@ -19,9 +19,13 @@
 
 package org.elasticsearch.index.rankeval;
 
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
 import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.search.builder.SearchSourceBuilder;
 
 import java.io.IOException;
@@ -36,21 +40,29 @@ import java.util.List;
  * */
 public class QuerySpec implements Writeable {
 
-    private int specId = 0;
+    private String specId;
     private SearchSourceBuilder testRequest;
     private List<String> indices = new ArrayList<>();
     private List<String> types = new ArrayList<>();
+    /** Collection of rated queries for this query QA specification.*/
+    private List<RatedDocument> ratedDocs = new ArrayList<>();
 
-    public QuerySpec(
+    public QuerySpec() {
-            int specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types) {
+        // ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser
+        // TODO decide if we can require only id as mandatory, set default values for the rest?
+    }
+
+    public QuerySpec(String specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types,
+            List<RatedDocument> ratedDocs) {
         this.specId = specId;
         this.testRequest = testRequest;
         this.indices = indices;
         this.types = types;
+        this.ratedDocs = ratedDocs;
     }
 
     public QuerySpec(StreamInput in) throws IOException {
-        this.specId = in.readInt();
+        this.specId = in.readString();
         testRequest = new SearchSourceBuilder(in);
         int indicesSize = in.readInt();
         indices = new ArrayList<String>(indicesSize);
@@ -62,11 +74,16 @@ public class QuerySpec implements Writeable {
         for (int i = 0; i < typesSize; i++) {
             this.types.add(in.readString());
         }
+        int intentSize = in.readInt();
+        ratedDocs = new ArrayList<>(intentSize);
+        for (int i = 0; i < intentSize; i++) {
+            ratedDocs.add(new RatedDocument(in));
+        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeInt(specId);
+        out.writeString(specId);
         testRequest.writeTo(out);
         out.writeInt(indices.size());
         for (String index : indices) {
@@ -76,6 +93,10 @@ public class QuerySpec implements Writeable {
         for (String type : types) {
             out.writeString(type);
         }
+        out.writeInt(ratedDocs.size());
+        for (RatedDocument ratedDoc : ratedDocs) {
+            ratedDoc.writeTo(out);
+        }
     }
 
     public SearchSourceBuilder getTestRequest() {
@@ -103,12 +124,69 @@ public class QuerySpec implements Writeable {
     }
 
     /** Returns a user supplied spec id for easier referencing. */
-    public int getSpecId() {
+    public String getSpecId() {
         return specId;
     }
 
     /** Sets a user supplied spec id for easier referencing. */
-    public void setSpecId(int specId) {
+    public void setSpecId(String specId) {
         this.specId = specId;
     }
+
+    /** Returns a list of rated documents to evaluate. */
+    public List<RatedDocument> getRatedDocs() {
+        return ratedDocs;
+    }
+
+    /** Set a list of rated documents for this query. */
+    public void setRatedDocs(List<RatedDocument> ratedDocs) {
+        this.ratedDocs = ratedDocs;
+    }
+
+    private static final ParseField ID_FIELD = new ParseField("id");
+    private static final ParseField REQUEST_FIELD = new ParseField("request");
+    private static final ParseField RATINGS_FIELD = new ParseField("ratings");
+    private static final ObjectParser<QuerySpec, RankEvalContext> PARSER = new ObjectParser<>("requests", QuerySpec::new);
+
+    static {
+        PARSER.declareString(QuerySpec::setSpecId, ID_FIELD);
+        PARSER.declareObject(QuerySpec::setTestRequest, (p, c) -> {
+            try {
+                return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(),  c.getSuggesters());
+            } catch (IOException ex) {
+                throw new ParsingException(p.getTokenLocation(), "error parsing request", ex);
+            }
+        } , REQUEST_FIELD);
+        PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> {
+            try {
+                return RatedDocument.fromXContent(p);
+            } catch (IOException ex) {
+                throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex);
+            }
+        } , RATINGS_FIELD);
+    }
+
+    /**
+     * Parses {@link QuerySpec} from rest representation:
+     *
+     * Example:
+     *  {
+     *   "id": "coffee_query",
+     *   "request": {
+     *           "query": {
+     *               "bool": {
+     *                   "must": [
+     *                       {"match": {"beverage": "coffee"}},
+     *                       {"term": {"browser": {"value": "safari"}}},
+     *                       {"term": {"time_of_day": {"value": "morning","boost": 2}}},
+     *                       {"term": {"ip_location": {"value": "ams","boost": 10}}}]}
+     *           },
+     *           "size": 10
+     *   },
+     *   "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ]
+     *  }
+     */
+    public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException {
+        return PARSER.parse(parser, context);
+    }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
+
+public class RankEvalContext implements ParseFieldMatcherSupplier {
+
+    private final ParseFieldMatcher parseFieldMatcher;
+    private final AggregatorParsers aggs;
+    private final Suggesters suggesters;
+    private final QueryParseContext parseContext;
+
+    public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, AggregatorParsers aggs,
+            Suggesters suggesters) {
+        this.parseFieldMatcher = parseFieldMatcher;
+        this.aggs = aggs;
+        this.suggesters = suggesters;
+        this.parseContext = parseContext;
+    }
+
+    public Suggesters getSuggesters() {
+        return this.suggesters;
+    }
+
+    public AggregatorParsers getAggs() {
+        return this.aggs;
+    }
+
+    @Override
+    public ParseFieldMatcher getParseFieldMatcher() {
+        return this.parseFieldMatcher;
+    }
+
+    public XContentParser parser() {
+        return this.parseContext.parser();
+    }
+
+    public QueryParseContext getParseContext() {
+        return this.parseContext;
+    }
+
+}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java

@@ -41,14 +41,18 @@ public class RankEvalRequest extends ActionRequest<RankEvalRequest> {
         return null; // TODO
     }
 
-    /** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics
+    /**
-     * to compute. */
+     * Returns the specification of this qa run including intents to execute,
+     * specifications detailing intent translation and metrics to compute.
+     */
     public RankEvalSpec getRankEvalSpec() {
         return task;
     }
 
-    /** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics
+    /**
-     * to compute. */
+     * Returns the specification of this qa run including intents to execute,
+     * specifications detailing intent translation and metrics to compute.
+     */
     public void setRankEvalSpec(RankEvalSpec task) {
         this.task = task;
     }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java

@@ -22,9 +22,10 @@ package org.elasticsearch.index.rankeval;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.io.stream.StreamInput;
 import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Map;
 
@@ -38,38 +39,48 @@ import java.util.Map;
  * by the search are not taken into consideration when computing precision at n - they are ignored.
  *
  **/
-public class RankEvalResponse extends ActionResponse {
+public class RankEvalResponse extends ActionResponse implements ToXContent {
 
-    private Collection<RankEvalResult> qualityResults = new ArrayList<>();
+    private RankEvalResult qualityResult;
 
     public RankEvalResponse() {
 
     }
 
     public RankEvalResponse(StreamInput in) throws IOException {
-        int size = in.readInt();
+        super.readFrom(in);
-        qualityResults = new ArrayList<>(size);
+        this.qualityResult = new RankEvalResult(in);
-        for (int i = 0; i < size; i++) {
-            qualityResults.add(new RankEvalResult(in));
-        }
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
         super.writeTo(out);
-        out.writeInt(qualityResults.size());
+        qualityResult.writeTo(out);
-        for (RankEvalResult result : qualityResults) {
-            result.writeTo(out);
-        }
     }
 
-    public void addRankEvalResult(int specId, double quality, Map<Integer, Collection<String>> unknownDocs) {
+    public void setRankEvalResult(RankEvalResult result) {
-        RankEvalResult result = new RankEvalResult(specId, quality, unknownDocs);
+        this.qualityResult = result;
-        this.qualityResults.add(result);
     }
 
-    public Collection<RankEvalResult> getRankEvalResults() {
+    public RankEvalResult getRankEvalResult() {
-        return qualityResults;
+        return qualityResult;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject("rank_eval");
+        builder.field("spec_id", qualityResult.getSpecId());
+        builder.field("quality_level", qualityResult.getQualityLevel());
+        builder.startArray("unknown_docs");
+        Map<String, Collection<String>> unknownDocs = qualityResult.getUnknownDocs();
+        for (String key : unknownDocs.keySet()) {
+            builder.startObject();
+            builder.field(key, unknownDocs.get(key));
+            builder.endObject();
+        }
+        builder.endArray();
+        builder.endObject();
+        return builder;
     }
 
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java

@@ -31,22 +31,23 @@ import java.util.Map;
  * For each precision at n computation the id of the search request specification used to generate search requests is returned
  * for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned.
  * */
+// TODO do we need an extra class for this or it RankEvalResponse enough?
 public class RankEvalResult implements Writeable {
-    /**ID of specification this result was generated for.*/
+    /**ID of QA specification this result was generated for.*/
-    private int specId;
+    private String specId;
-    /**Average precision observed when issueing query intents with this spec.*/
+    /**Average precision observed when issuing query intents with this specification.*/
     private double qualityLevel;
     /**Mapping from intent id to all documents seen for this intent that were not annotated.*/
-    private Map<Integer, Collection<String>> unknownDocs;
+    private Map<String, Collection<String>> unknownDocs;
 
     @SuppressWarnings("unchecked")
     public RankEvalResult(StreamInput in) throws IOException {
-        this.specId = in.readInt();
+        this.specId = in.readString();
         this.qualityLevel = in.readDouble();
-        this.unknownDocs = (Map<Integer, Collection<String>>) in.readGenericValue();
+        this.unknownDocs = (Map<String, Collection<String>>) in.readGenericValue();
     }
 
-    public RankEvalResult(int specId, double quality, Map<Integer, Collection<String>> unknownDocs) {
+    public RankEvalResult(String specId, double quality, Map<String, Collection<String>> unknownDocs) {
         this.specId = specId;
         this.qualityLevel = quality;
         this.unknownDocs = unknownDocs;
@@ -54,12 +55,12 @@ public class RankEvalResult implements Writeable {
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeInt(specId);
+        out.writeString(specId);
         out.writeDouble(qualityLevel);
         out.writeGenericValue(getUnknownDocs());
     }
 
-    public int getSpecId() {
+    public String getSpecId() {
         return specId;
     }
 
@@ -67,7 +68,12 @@ public class RankEvalResult implements Writeable {
         return qualityLevel;
     }
 
-    public Map<Integer, Collection<String>> getUnknownDocs() {
+    public Map<String, Collection<String>> getUnknownDocs() {
         return unknownDocs;
     }
+
+    @Override
+    public String toString() {
+        return "RankEvalResult, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs;
+    }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java

@@ -35,45 +35,54 @@ import java.util.Collection;
  * */
 
 public class RankEvalSpec implements Writeable {
-    /** Collection of query intents to check against including expected document ids.*/
+
-    private Collection<RatedQuery> intents = new ArrayList<>();
     /** Collection of query specifications, that is e.g. search request templates to use for query translation. */
     private Collection<QuerySpec> specifications = new ArrayList<>();
-    /** Definition of n in precision at n */
+    /** Definition of the quality metric, e.g. precision at N */
     private RankedListQualityMetric eval;
+    /** a unique id for the whole QA task */
+    private String taskId;
 
+    public RankEvalSpec() {
+        // TODO think if no args ctor is okay
+    }
 
-    public RankEvalSpec(Collection<RatedQuery> intents, Collection<QuerySpec> specs, RankedListQualityMetric metric) {
+    public RankEvalSpec(String taskId, Collection<QuerySpec> specs, RankedListQualityMetric metric) {
-        this.intents = intents;
+        this.taskId = taskId;
         this.specifications = specs;
         this.eval = metric;
     }
 
     public RankEvalSpec(StreamInput in) throws IOException {
-        int intentSize = in.readInt();
-        intents = new ArrayList<>(intentSize);
-        for (int i = 0; i < intentSize; i++) {
-           intents.add(new RatedQuery(in));
-        }
         int specSize = in.readInt();
         specifications = new ArrayList<>(specSize);
         for (int i = 0; i < specSize; i++) {
             specifications.add(new QuerySpec(in));
         }
         eval = in.readNamedWriteable(RankedListQualityMetric.class); // TODO add to registry
+        taskId = in.readString();
     }
 
     @Override
     public void writeTo(StreamOutput out) throws IOException {
-        out.writeInt(intents.size());
-        for (RatedQuery query : intents) {
-            query.writeTo(out);
-        }
         out.writeInt(specifications.size());
         for (QuerySpec spec : specifications) {
             spec.writeTo(out);
         }
         out.writeNamedWriteable(eval);
+        out.writeString(taskId);
+    }
+
+    public void setEval(RankedListQualityMetric eval) {
+        this.eval = eval;
+    }
+
+    public void setTaskId(String taskId) {
+        this.taskId = taskId;
+    }
+
+    public String getTaskId() {
+        return this.taskId;
     }
 
     /** Returns the precision at n configuration (containing level of n to consider).*/
@@ -86,16 +95,6 @@ public class RankEvalSpec implements Writeable {
         this.eval = config;
     }
 
-    /** Returns a list of search intents to evaluate. */
-    public Collection<RatedQuery> getIntents() {
-        return intents;
-    }
-
-    /** Set a list of search intents to evaluate. */
-    public void setIntents(Collection<RatedQuery> intents) {
-        this.intents = intents;
-    }
-
     /** Returns a list of intent to query translation specifications to evaluate. */
     public Collection<QuerySpec> getSpecifications() {
         return specifications;

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java

@@ -19,15 +19,23 @@
 
 package org.elasticsearch.index.rankeval;
 
+import org.elasticsearch.common.ParseFieldMatcherSupplier;
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.NamedWriteable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
 import org.elasticsearch.search.SearchHit;
 
+import java.io.IOException;
+import java.util.List;
+
 /**
  * Classes implementing this interface provide a means to compute the quality of a result list
  * returned by some search.
  *
  * RelevancyLevel specifies the type of object determining the relevancy level of some known docid.
  * */
-public interface RankedListQualityMetric extends Evaluator {
+public abstract class RankedListQualityMetric implements NamedWriteable {
 
     /**
      * Returns a single metric representing the ranking quality of a set of returned documents
@@ -36,6 +44,27 @@ public interface RankedListQualityMetric extends Evaluator {
      * @param hits the result hits as returned by some search
      * @return some metric representing the quality of the result hit list wrt. to relevant doc ids.
      * */
-    @Override
+    public abstract EvalQueryQuality evaluate(SearchHit[] hits, List<RatedDocument> ratedDocs);
-    EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent);
+
+    public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
+        RankedListQualityMetric rc;
+        Token token = parser.nextToken();
+        if (token != XContentParser.Token.FIELD_NAME) {
+            throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name");
+        }
+        String metricName = parser.currentName();
+
+        switch (metricName) {
+        case PrecisionAtN.NAME:
+            rc = PrecisionAtN.fromXContent(parser, context);
+            break;
+        default:
+            throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName);
+        }
+        if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
+            // if we are at END_OBJECT, move to the next one...
+            parser.nextToken();
+        }
+        return rc;
+    }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java

@@ -0,0 +1,86 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.ParsingException;
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.io.stream.Writeable;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+
+import java.io.IOException;
+
+/**
+ * A document ID and its rating for the query QA use case.
+ * */
+public class RatedDocument implements Writeable {
+
+    private final String docId;
+    private final int rating;
+
+    public RatedDocument(String docId, int rating) {
+        this.docId = docId;
+        this.rating = rating;
+    }
+
+    public RatedDocument(StreamInput in) throws IOException {
+        this.docId = in.readString();
+        this.rating = in.readVInt();
+    }
+
+    public String getDocID() {
+        return docId;
+    }
+
+    public int getRating() {
+        return rating;
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        out.writeString(docId);
+        out.writeVInt(rating);
+    }
+
+    public static RatedDocument fromXContent(XContentParser parser) throws IOException {
+        String id = null;
+        int rating = Integer.MIN_VALUE;
+        Token token;
+        while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
+            if (parser.currentToken().equals(Token.FIELD_NAME)) {
+                if (id != null) {
+                    throw new ParsingException(parser.getTokenLocation(), "only one document id allowed, found [{}] but already got [{}]",
+                            parser.currentName(), id);
+                }
+                id = parser.currentName();
+            } else if (parser.currentToken().equals(Token.VALUE_NUMBER)) {
+                rating = parser.intValue();
+            } else {
+                throw new ParsingException(parser.getTokenLocation(), "unexpected token [{}] while parsing rated document",
+                        token);
+            }
+        }
+        if (id ==  null) {
+            throw new ParsingException(parser.getTokenLocation(), "didn't find document id");
+        }
+        return new RatedDocument(id, rating);
+    }
+}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java

@@ -1,92 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.index.rankeval;
-
-import org.elasticsearch.common.io.stream.StreamInput;
-import org.elasticsearch.common.io.stream.StreamOutput;
-import org.elasticsearch.common.io.stream.Writeable;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
-
-/**
- * Objects of this class represent one type of user query to qa. Each query comprises a user supplied id for easer referencing,
- * a set of parameters as supplied by the end user to the search application as well as a set of rated documents (ratings e.g.
- * supplied by manual result tagging or some form of automated click log based process).
- * */
-public class RatedQuery implements Writeable {
-    
-    private final int intentId;
-    private final Map<String, Object> intentParameters;
-    private final Map<String, Integer> ratedDocuments;
-
-    public RatedQuery(
-            int intentId, Map<String, Object> intentParameters, Map<String, Integer> ratedDocuments) {
-        this.intentId = intentId;
-        this.intentParameters = intentParameters;
-        this.ratedDocuments = ratedDocuments;
-    }
-
-    public RatedQuery(StreamInput in) throws IOException {
-        this.intentId = in.readInt();
-        this.intentParameters = in.readMap();
-        
-        int ratedDocsSize = in.readInt();
-        this.ratedDocuments = new HashMap<>(ratedDocsSize);
-        for (int i = 0; i < ratedDocsSize; i++) {
-            this.ratedDocuments.put(in.readString(), in.readInt());
-        }
-    }
-
-    @Override
-    public void writeTo(StreamOutput out) throws IOException {
-        out.writeInt(intentId);
-        out.writeMap(intentParameters);
-        out.writeInt(ratedDocuments.size());
-        for(Entry<String, Integer> entry : ratedDocuments.entrySet()) {
-            out.writeString(entry.getKey());
-            out.writeInt(entry.getValue());
-        }
-    }
-    
-    /** For easier referencing users are allowed to supply unique ids with each search intent they want to check for
-     * performance quality wise.*/
-    public int getIntentId() {
-        return intentId;
-    }
-
-    
-    /**
-     * Returns a mapping from query parameter name to real parameter - ideally as parsed from real user logs.
-     * */
-    public Map<String, Object> getIntentParameters() {
-        return intentParameters;
-    }
-
-    /**
-     * Returns a set of documents and their ratings as supplied by the users.
-     * */
-    public Map<String, Integer> getRatedDocuments() {
-        return ratedDocuments;
-    }
-
-}

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java

@@ -20,21 +20,29 @@
 package org.elasticsearch.index.rankeval;
 
 import org.elasticsearch.client.node.NodeClient;
-import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.ParseField;
+import org.elasticsearch.common.ParsingException;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.inject.Inject;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
 import org.elasticsearch.indices.query.IndicesQueriesRegistry;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestChannel;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
 import org.elasticsearch.rest.action.support.RestActions;
+import org.elasticsearch.rest.action.support.RestToXContentListener;
 import org.elasticsearch.search.aggregations.AggregatorParsers;
 import org.elasticsearch.search.suggest.Suggesters;
 
 import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
 
 import static org.elasticsearch.rest.RestRequest.Method.GET;
 import static org.elasticsearch.rest.RestRequest.Method.POST;
@@ -45,7 +53,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
  * General Format:
  *
  *
-  { "requests": [{
+  {
+    "spec_id": "human_readable_id",
+    "requests": [{
         "id": "human_readable_id",
         "request": { ... request to check ... },
         "ratings": { ... mapping from doc id to rating value ... }
@@ -53,12 +63,15 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
     "metric": {
         "... metric_name... ": {
             "... metric_parameter_key ...": ...metric_parameter_value...
-        }}}
+        }
+    }
+  }
  *
  * Example:
  *
  *
-   {"requests": [{
+   {"spec_id": "huge_weight_on_location",
+    "requests": [{
         "id": "amsterdam_query",
         "request": {
                 "query": {
@@ -78,6 +91,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
             "3": 1,
             "4": 1
         }
+        }
     }, {
         "id": "berlin_query",
         "request": {
@@ -100,7 +114,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
     }],
     "metric": {
         "precisionAtN": {
-            "size": 10}}
+            "size": 10
+            }
+    }
   }
 
  *
@@ -135,7 +151,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
         "failed": 0
     },
     "rank_eval": [{
-        "spec_id": "huge_weight_on_city",
+        "spec_id": "huge_weight_on_location",
         "quality_level": 0.4,
         "unknown_docs": [{
             "amsterdam_query": [5, 10, 23]
@@ -149,10 +165,17 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
  * */
 public class RestRankEvalAction extends BaseRestHandler {
 
+    private IndicesQueriesRegistry queryRegistry;
+    private AggregatorParsers aggregators;
+    private Suggesters suggesters;
+
     @Inject
     public RestRankEvalAction(Settings settings, RestController controller, IndicesQueriesRegistry queryRegistry,
             AggregatorParsers aggParsers, Suggesters suggesters) {
         super(settings);
+        this.queryRegistry = queryRegistry;
+        this.aggregators = aggParsers;
+        this.suggesters = suggesters;
         controller.registerHandler(GET, "/_rank_eval", this);
         controller.registerHandler(POST, "/_rank_eval", this);
         controller.registerHandler(GET, "/{index}/_rank_eval", this);
@@ -164,22 +187,47 @@ public class RestRankEvalAction extends BaseRestHandler {
     @Override
     public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
         RankEvalRequest rankEvalRequest = new RankEvalRequest();
-        //parseRankEvalRequest(rankEvalRequest, request, parseFieldMatcher);
+        BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null;
-        //client.rankEval(rankEvalRequest, new RestStatusToXContentListener<>(channel));
+        try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
-    }
+            QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
-
-    public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, ParseFieldMatcher parseFieldMatcher)
-        throws IOException {
-
-        String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
-        BytesReference restContent = null;
-        if (restContent == null) {
-            if (RestActions.hasBodyContent(request)) {
-                restContent = RestActions.getRestContent(request);
-            }
-        }
             if (restContent != null) {
+                parseRankEvalRequest(rankEvalRequest, request,
+                        new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters));
+            }
+        }
+        client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener<RankEvalResponse>(channel));
     }
 
+    private static final ParseField SPECID_FIELD = new ParseField("spec_id");
+    private static final ParseField METRIC_FIELD = new ParseField("metric");
+    private static final ParseField REQUESTS_FIELD = new ParseField("requests");
+    private static final ObjectParser<RankEvalSpec, RankEvalContext> PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new);
+
+    static {
+        PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD);
+        PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> {
+            try {
+                return RankedListQualityMetric.fromXContent(p, c);
+            } catch (IOException ex) {
+                throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex);
+            }
+        } , METRIC_FIELD);
+        PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> {
+            try {
+                return QuerySpec.fromXContent(p, c);
+            } catch (IOException ex) {
+                throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex);
+            }
+        } , REQUESTS_FIELD);
+    }
+
+    public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context)
+            throws IOException {
+        List<String> indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
+        RankEvalSpec spec = PARSER.parse(context.parser(), context);
+        for (QuerySpec specification : spec.getSpecifications()) {
+            specification.setIndices(indices);
+        };
+        rankEvalRequest.setRankEvalSpec(spec);
     }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java

@@ -75,46 +75,36 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
         this.clusterService = clusterService;
         this.actionFilters = actionFilters;
 
+        // TODO this should maybe move to some registry on startup
         namedWriteableRegistry.register(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new);
     }
 
     @Override
     protected void doExecute(RankEvalRequest request, ActionListener<RankEvalResponse> listener) {
-        RankEvalResponse response = new RankEvalResponse();
         RankEvalSpec qualityTask = request.getRankEvalSpec();
         RankedListQualityMetric metric = qualityTask.getEvaluator();
 
-        for (QuerySpec spec : qualityTask.getSpecifications()) {
         double qualitySum = 0;
-
+        Map<String, Collection<String>> unknownDocs = new HashMap<String, Collection<String>>();
+        Collection<QuerySpec> specifications = qualityTask.getSpecifications();
+        for (QuerySpec spec : specifications) {
             SearchSourceBuilder specRequest = spec.getTestRequest();
             String[] indices = new String[spec.getIndices().size()];
             spec.getIndices().toArray(indices);
             SearchRequest templatedRequest = new SearchRequest(indices, specRequest);
 
-
+            TransportSearchAction transportSearchAction = new TransportSearchAction(settings, threadPool, searchPhaseController,
-            Map<Integer, Collection<String>> unknownDocs = new HashMap<Integer, Collection<String>>();
+                    transportService, searchTransportService, clusterService, actionFilters, indexNameExpressionResolver);
-            Collection<RatedQuery> intents = qualityTask.getIntents();
-            for (RatedQuery intent : intents) {
-
-                TransportSearchAction transportSearchAction = new TransportSearchAction(
-                        settings, 
-                        threadPool, 
-                        searchPhaseController, 
-                        transportService, 
-                        searchTransportService, 
-                        clusterService, 
-                        actionFilters, 
-                        indexNameExpressionResolver);
             ActionFuture<SearchResponse> searchResponse = transportSearchAction.execute(templatedRequest);
             SearchHits hits = searchResponse.actionGet().getHits();
 
-                EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), intent);
+            EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), spec.getRatedDocs());
             qualitySum += intentQuality.getQualityLevel();
-                unknownDocs.put(intent.getIntentId(), intentQuality.getUnknownDocs());
+            unknownDocs.put(spec.getSpecId(), intentQuality.getUnknownDocs());
-            }
-            response.addRankEvalResult(spec.getSpecId(), qualitySum / intents.size(), unknownDocs);
         }
+        RankEvalResponse response = new RankEvalResponse();
+        RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), qualitySum / specifications.size(), unknownDocs);
+        response.setRankEvalResult(result);
         listener.onResponse(response);
     }
 }

modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java

@@ -1,170 +0,0 @@
-/*
- * Licensed to Elasticsearch under one or more contributor
- * license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright
- * ownership. Elasticsearch licenses this file to you under
- * the Apache License, Version 2.0 (the "License"); you may
- * not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *    http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.elasticsearch.action.quality;
-
-import org.elasticsearch.action.search.SearchResponse;
-import org.elasticsearch.index.query.MatchQueryBuilder;
-import org.elasticsearch.index.rankeval.PrecisionAtN;
-import org.elasticsearch.index.rankeval.RankEvalPlugin;
-import org.elasticsearch.index.rankeval.RatedQuery;
-import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
-import org.elasticsearch.plugins.Plugin;
-import org.elasticsearch.search.SearchHit;
-import org.elasticsearch.test.ESIntegTestCase;
-import org.junit.Before;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.concurrent.ExecutionException;
-
-@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0)
-// NORELEASE need to fix transport client use case
-public class PrecisionAtRequestTests  extends ESIntegTestCase {
-    @Override
-    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return pluginList(RankEvalPlugin.class);
-    }
-    
-    @Override
-    protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return pluginList(RankEvalPlugin.class);
-    }
-
-    @Before
-    public void setup() {
-        createIndex("test");
-        ensureGreen();
-
-        client().prepareIndex("test", "testtype").setId("1")
-                .setSource("text", "berlin").get();
-        client().prepareIndex("test", "testtype").setId("2")
-                .setSource("text", "amsterdam").get();
-        client().prepareIndex("test", "testtype").setId("3")
-                .setSource("text", "amsterdam").get();
-        client().prepareIndex("test", "testtype").setId("4")
-                .setSource("text", "amsterdam").get();
-        client().prepareIndex("test", "testtype").setId("5")
-                .setSource("text", "amsterdam").get();
-        client().prepareIndex("test", "testtype").setId("6")
-                .setSource("text", "amsterdam").get();
-        refresh();
-    }
-
-
-    public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException {
-        // TODO turn into unit test - no need to execute the query here to fill hits object
-        MatchQueryBuilder query = new MatchQueryBuilder("text", "berlin");
-
-        SearchResponse response = client().prepareSearch().setQuery(query)
-                .execute().actionGet();
-
-        Map<String, Integer> relevant = new HashMap<>();
-        relevant.put("1", Rating.RELEVANT.ordinal());
-        RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant);
-        SearchHit[] hits = response.getHits().getHits();
-
-        assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001);
-    }
-    
-    public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException {
-        // TODO turn into unit test - no need to actually execute the query here to fill the hits object
-        MatchQueryBuilder query = new MatchQueryBuilder("text", "amsterdam");
-
-        SearchResponse response = client().prepareSearch().setQuery(query)
-                .execute().actionGet();
-
-        Map<String, Integer> relevant = new HashMap<>();
-        relevant.put("2", Rating.RELEVANT.ordinal());
-        relevant.put("3", Rating.RELEVANT.ordinal());
-        relevant.put("4", Rating.RELEVANT.ordinal());
-        relevant.put("5", Rating.RELEVANT.ordinal());
-        relevant.put("6",  Rating.IRRELEVANT.ordinal());
-        RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant);
-        SearchHit[] hits = response.getHits().getHits();
-
-        assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001);
-    }
-    
-    public void testPrecisionJSON() {
-        
-    }
-    
-/*    public void testPrecisionAction() {
-        // TODO turn into REST test?
-
-        Collection<RatedQuery> intents = new ArrayList<RatedQuery>();
-        RatedQuery intentAmsterdam = new RatedQuery(
-                0,
-                createParams("var", "amsterdam"),
-                createRelevant("2", "3", "4", "5"));
-        intents.add(intentAmsterdam);
-
-        RatedQuery intentBerlin = new RatedQuery(
-                1,
-                createParams("var", "berlin"),
-                createRelevant("1"));
-        intents.add(intentBerlin);
-
-        Collection<QuerySpec> specs = new ArrayList<QuerySpec>();
-        ArrayList<String> indices = new ArrayList<>();
-        indices.add("test");
-        ArrayList<String> types = new ArrayList<>();
-        types.add("testtype");
-
-        SearchSourceBuilder source = new SearchSourceBuilder();
-        QuerySpec spec = new QuerySpec(0, source, indices, types);
-        specs.add(spec);
-
-        RankEvalSpec task = new RankEvalSpec(intents, specs, new PrecisionAtN(10));
-        
-        RankEvalRequestBuilder builder = new RankEvalRequestBuilder(
-                client(),
-                RankEvalAction.INSTANCE,
-                new RankEvalRequest());
-        builder.setRankEvalSpec(task);
-
-        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
-        RankEvalResult result = response.getRankEvalResults().iterator().next();
-        for (Entry<Integer, Collection<String>> entry : result.getUnknownDocs().entrySet()) {
-            if (entry.getKey() == 0) {
-                assertEquals(1, entry.getValue().size());
-            } else {
-                assertEquals(0, entry.getValue().size());
-            }
-        }
-    }*/
-    
-    private Map<String, Integer> createRelevant(String... docs) {
-        Map<String, Integer> relevant = new HashMap<>();
-        for (String doc : docs) {
-            relevant.put(doc, Rating.RELEVANT.ordinal());
-        }
-        return relevant;
-    }
-    
-    private Map<String, Object> createParams(String key, String value) {
-        Map<String, Object> parameters = new HashMap<>();
-        parameters.put(key, value);
-        return parameters;
-    }
-
- }

modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java

@@ -0,0 +1,122 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.action.quality;
+
+import org.elasticsearch.index.query.MatchAllQueryBuilder;
+import org.elasticsearch.index.rankeval.PrecisionAtN;
+import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
+import org.elasticsearch.index.rankeval.QuerySpec;
+import org.elasticsearch.index.rankeval.RankEvalAction;
+import org.elasticsearch.index.rankeval.RankEvalPlugin;
+import org.elasticsearch.index.rankeval.RankEvalRequest;
+import org.elasticsearch.index.rankeval.RankEvalRequestBuilder;
+import org.elasticsearch.index.rankeval.RankEvalResponse;
+import org.elasticsearch.index.rankeval.RankEvalResult;
+import org.elasticsearch.index.rankeval.RankEvalSpec;
+import org.elasticsearch.index.rankeval.RatedDocument;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.search.builder.SearchSourceBuilder;
+import org.elasticsearch.test.ESIntegTestCase;
+import org.junit.Before;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0)
+// NORELEASE need to fix transport client use case
+public class RankEvalRequestTests  extends ESIntegTestCase {
+    @Override
+    protected Collection<Class<? extends Plugin>> transportClientPlugins() {
+        return pluginList(RankEvalPlugin.class);
+    }
+
+    @Override
+    protected Collection<Class<? extends Plugin>> nodePlugins() {
+        return pluginList(RankEvalPlugin.class);
+    }
+
+    @Before
+    public void setup() {
+        createIndex("test");
+        ensureGreen();
+
+        client().prepareIndex("test", "testtype").setId("1")
+                .setSource("text", "berlin").get();
+        client().prepareIndex("test", "testtype").setId("2")
+                .setSource("text", "amsterdam").get();
+        client().prepareIndex("test", "testtype").setId("3")
+                .setSource("text", "amsterdam").get();
+        client().prepareIndex("test", "testtype").setId("4")
+                .setSource("text", "amsterdam").get();
+        client().prepareIndex("test", "testtype").setId("5")
+                .setSource("text", "amsterdam").get();
+        client().prepareIndex("test", "testtype").setId("6")
+                .setSource("text", "amsterdam").get();
+        refresh();
+    }
+
+    public void testPrecisionAtRequest() {
+        ArrayList<String> indices = new ArrayList<>();
+        indices.add("test");
+        ArrayList<String> types = new ArrayList<>();
+        types.add("testtype");
+
+        String specId = randomAsciiOfLength(10);
+        List<QuerySpec> specifications = new ArrayList<>();
+        SearchSourceBuilder testQuery = new SearchSourceBuilder();
+        testQuery.query(new MatchAllQueryBuilder());
+        specifications.add(new QuerySpec("amsterdam_query",  testQuery, indices, types, createRelevant("2", "3", "4", "5")));
+        specifications.add(new QuerySpec("berlin_query",  testQuery, indices, types, createRelevant("1")));
+
+        RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10));
+
+        RankEvalRequestBuilder builder = new RankEvalRequestBuilder(
+                client(),
+                RankEvalAction.INSTANCE,
+                new RankEvalRequest());
+        builder.setRankEvalSpec(task);
+
+        RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
+        RankEvalResult result = response.getRankEvalResult();
+        assertEquals(specId, result.getSpecId());
+        assertEquals(1.0, result.getQualityLevel(), Double.MIN_VALUE);
+        Set<Entry<String, Collection<String>>> entrySet = result.getUnknownDocs().entrySet();
+        assertEquals(2, entrySet.size());
+        for (Entry<String, Collection<String>> entry : entrySet) {
+            if (entry.getKey() == "amsterdam_query") {
+                assertEquals(2, entry.getValue().size());
+            }
+            if (entry.getKey() == "berlin_query") {
+                assertEquals(5, entry.getValue().size());
+            }
+        }
+    }
+
+    private static List<RatedDocument> createRelevant(String... docs) {
+        List<RatedDocument> relevant = new ArrayList<>();
+        for (String doc : docs) {
+            relevant.add(new RatedDocument(doc, Rating.RELEVANT.ordinal()));
+        }
+        return relevant;
+    }
+ }

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java

@@ -0,0 +1,69 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.text.Text;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
+import org.elasticsearch.search.SearchHit;
+import org.elasticsearch.search.internal.InternalSearchHit;
+import org.elasticsearch.test.ESTestCase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+public class PrecisionAtNTests extends ESTestCase {
+
+    public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException {
+        List<RatedDocument> rated = new ArrayList<>();
+        rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal()));
+        SearchHit[] hits = new InternalSearchHit[1];
+        hits[0] = new InternalSearchHit(0, "0", new Text("type"), Collections.emptyMap());
+        assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001);
+    }
+
+    public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException {
+        List<RatedDocument> rated = new ArrayList<>();
+        rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal()));
+        rated.add(new RatedDocument("1", Rating.RELEVANT.ordinal()));
+        rated.add(new RatedDocument("2", Rating.RELEVANT.ordinal()));
+        rated.add(new RatedDocument("3", Rating.RELEVANT.ordinal()));
+        rated.add(new RatedDocument("4", Rating.IRRELEVANT.ordinal()));
+        SearchHit[] hits = new InternalSearchHit[5];
+        for (int i = 0; i < 5; i++) {
+            hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap());
+        }
+        assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001);
+    }
+
+    public void testParseFromXContent() throws IOException {
+        String xContent = " {\n"
+         + "   \"size\": 10\n"
+         + "}";
+        XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent);
+        PrecisionAtN precicionAt = PrecisionAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT);
+        assertEquals(10, precicionAt.getN());
+    }
+}

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java

@@ -0,0 +1,98 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.rankeval;
+
+import org.elasticsearch.common.ParseFieldMatcher;
+import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.xcontent.ParseFieldRegistry;
+import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.index.query.QueryParseContext;
+import org.elasticsearch.indices.query.IndicesQueriesRegistry;
+import org.elasticsearch.search.SearchModule;
+import org.elasticsearch.search.aggregations.AggregatorParsers;
+import org.elasticsearch.search.suggest.Suggesters;
+import org.elasticsearch.test.ESTestCase;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.IOException;
+import java.util.List;
+
+public class QuerySpecTests extends ESTestCase {
+
+    private static IndicesQueriesRegistry queriesRegistry;
+    private static SearchModule searchModule;
+    private static Suggesters suggesters;
+    private static AggregatorParsers aggsParsers;
+
+    /**
+    * setup for the whole base test class
+    */
+    @BeforeClass
+    public static void init() throws IOException {
+        aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), new ParseFieldRegistry<>("aggregation_pipes"));
+        searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
+        queriesRegistry = searchModule.getQueryParserRegistry();
+        suggesters = searchModule.getSuggesters();
+    }
+
+    @AfterClass
+    public static void afterClass() throws Exception {
+        queriesRegistry = null;
+        searchModule = null;
+        suggesters = null;
+        aggsParsers = null;
+    }
+
+    public void testParseFromXContent() throws IOException {
+        String querySpecString = " {\n"
+         + "   \"id\": \"my_qa_query\",\n"
+         + "   \"request\": {\n"
+         + "           \"query\": {\n"
+         + "               \"bool\": {\n"
+         + "                   \"must\": [\n"
+         + "                       {\"match\": {\"beverage\": \"coffee\"}},\n"
+         + "                       {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n"
+         + "                       {\"term\": {\"time_of_day\": {\"value\": \"morning\",\"boost\": 2}}},\n"
+         + "                       {\"term\": {\"ip_location\": {\"value\": \"ams\",\"boost\": 10}}}]}\n"
+         + "           },\n"
+         + "           \"size\": 10\n"
+         + "   },\n"
+         + "   \"ratings\": [ {\"1\": 1 }, { \"2\": 0 }, { \"3\": 1 } ]\n"
+         + "}";
+        XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString);
+        QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT);
+        RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext,
+                aggsParsers, suggesters);
+        QuerySpec specification = QuerySpec.fromXContent(parser, rankContext);
+        assertEquals("my_qa_query", specification.getSpecId());
+        assertNotNull(specification.getTestRequest());
+        List<RatedDocument> ratedDocs = specification.getRatedDocs();
+        assertEquals(3, ratedDocs.size());
+        assertEquals("1", ratedDocs.get(0).getDocID());
+        assertEquals(1, ratedDocs.get(0).getRating());
+        assertEquals("2", ratedDocs.get(1).getDocID());
+        assertEquals(0, ratedDocs.get(1).getRating());
+        assertEquals("3", ratedDocs.get(2).getDocID());
+        assertEquals(1, ratedDocs.get(2).getRating());
+    }
+}

modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml

@@ -5,44 +5,53 @@
       index:
         index:   foo
         type:    bar
-        id:      1
+        id:      doc1
         body:    { "text": "berlin" }
 
   - do:
       index:
         index:  foo
         type:   bar
-        id:     2
+        id:     doc2
         body:   { "text": "amsterdam" }
 
   - do:
       index:
         index:  foo
         type:   bar
-        id:     3
+        id:     doc3
         body:   { "text": "amsterdam" }
         
+  - do:
+      index:
+        index:  foo
+        type:   bar
+        id:     doc4
+        body:   { "text": "something about amsterdam and berlin" }
+
   - do:
       indices.refresh: {}
 
   - do:
       rank_eval:
-        body:
+        body: {
-          requests: [
+          "spec_id" : "cities_qa_queries",
+          "requests" : [
             {
-                id: "amsterdam_query",
+                "id": "amsterdam_query",
-                request: { query: {match : {text : "amsterdam" }}},
+                "request": { "query": { "match" : {"text" : "amsterdam" }}},
-                ratings: { "1": 0, "2": 1, "3": 1 }
+                "ratings": [{ "doc1": 0}, {"doc2": 1}, {"doc3": 1}]
-            }, {
+            },
-                id: "berlin_query",
+            {
-                request: { query: { match : { text : "berlin" } }, size : 10 },
+                "id" : "berlin_query",
-                ratings: {"1": 1}
+                "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
+                "ratings": [{"doc1": 1}]
+            }
+          ],
+          "metric" : { "precisionatn": { "size": 10}}
         }
-          ]
-          metric: { precisionAtN: { size: 10}}
-
-  - match: {quality_level: 1}
-  - gte: { took: 0 }
-  - is_false: task
-  - is_false: deleted
 
+  - match: {rank_eval.spec_id: "cities_qa_queries"}
+  - match: {rank_eval.quality_level: 1}
+  - match: {rank_eval.unknown_docs.0.amsterdam_query:  [ "doc4"]}
+  - match: {rank_eval.unknown_docs.1.berlin_query:  [ "doc4"]}