Merge pull request #19623 from MaineC/feature/rank-eval

Add parsing of type information

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java

@@ -128,6 +128,7 @@ public class PrecisionAtN extends RankedListQualityMetric {
         return new EvalQueryQuality(precision, unknownDocIds);
     }
 
+    // TODO add abstraction that also works for other metrics
     public enum Rating {
         IRRELEVANT, RELEVANT;
     }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java

@@ -38,6 +38,7 @@ import java.util.Map;
  * Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned
  * by the search are not taken into consideration when computing precision at n - they are ignored.
  *
+ * TODO get rid of either this or RankEvalResult
  **/
 public class RankEvalResponse extends ActionResponse implements ToXContent {
 

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java

@@ -32,6 +32,7 @@ import java.util.Map;
  * for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned.
  * */
 // TODO do we need an extra class for this or it RankEvalResponse enough?
+// TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results
 public class RankEvalResult implements Writeable {
     /**ID of QA specification this result was generated for.*/
     private String specId;

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java

@@ -54,6 +54,7 @@ public abstract class RankedListQualityMetric implements NamedWriteable {
         }
         String metricName = parser.currentName();
 
+        // TODO maybe switch to using a plugable registry later?
         switch (metricName) {
         case PrecisionAtN.NAME:
             rc = PrecisionAtN.fromXContent(parser, context);

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java

@@ -33,6 +33,7 @@ import java.io.IOException;
  * */
 public class RatedDocument implements Writeable {
 
+    // TODO augment with index name and type name
     private final String docId;
     private final int rating;
 

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java

@@ -192,6 +192,7 @@ public class RestRankEvalAction extends BaseRestHandler {
             QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
             if (restContent != null) {
                 parseRankEvalRequest(rankEvalRequest, request,
+                        // TODO can we get rid of aggregators parsers and suggesters?
                         new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters));
             }
         }
@@ -224,10 +225,13 @@ public class RestRankEvalAction extends BaseRestHandler {
     public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context)
             throws IOException {
         List<String> indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
+        List<String> types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type")));
         RankEvalSpec spec = PARSER.parse(context.parser(), context);
         for (QuerySpec specification : spec.getSpecifications()) {
             specification.setIndices(indices);
+            specification.setTypes(types);
         };
+
         rankEvalRequest.setRankEvalSpec(spec);
     }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java

@@ -92,6 +92,9 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
             String[] indices = new String[spec.getIndices().size()];
             spec.getIndices().toArray(indices);
             SearchRequest templatedRequest = new SearchRequest(indices, specRequest);
+            String[] types = new String[spec.getTypes().size()];
+            spec.getTypes().toArray(types);
+            templatedRequest.types(types);
 
             TransportSearchAction transportSearchAction = new TransportSearchAction(settings, threadPool, searchPhaseController,
                     transportService, searchTransportService, clusterService, actionFilters, indexNameExpressionResolver);
@@ -103,6 +106,7 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
             unknownDocs.put(spec.getSpecId(), intentQuality.getUnknownDocs());
         }
         RankEvalResponse response = new RankEvalResponse();
+        // TODO move averaging to actual metric, also add other statistics
         RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), qualitySum / specifications.size(), unknownDocs);
         response.setRankEvalResult(result);
         listener.onResponse(response);

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java

@@ -65,6 +65,7 @@ public class QuerySpecTests extends ESTestCase {
         aggsParsers = null;
     }
 
+    // TODO add some sort of roundtrip testing like we have now for queries?
     public void testParseFromXContent() throws IOException {
         String querySpecString = " {\n"
          + "   \"id\": \"my_qa_query\",\n"