Remove unknown docs from EvalQueryQuality

The unknown document section in the response for each query can be rendered
using the rated hits that are now also part of the response by just filtering
the documents without a rating.

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java

@@ -36,7 +36,6 @@ import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
 
-import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments;
 import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings;
 
 public class DiscountedCumulativeGainAt implements RankedListQualityMetric {
@@ -159,7 +158,6 @@ public class DiscountedCumulativeGainAt implements RankedListQualityMetric {
         }
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg);
         evalQueryQuality.addHitsAndRatings(ratedHits);
-        evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedHits));
         return evalQueryQuality;
     }
 

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java

@@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
 
 import java.io.IOException;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Objects;;
 
@@ -39,7 +38,6 @@ import java.util.Objects;;
 public class EvalQueryQuality implements ToXContent, Writeable {
 
     /** documents seen as result for one request that were not annotated.*/
-    private List<DocumentKey> unknownDocs = new ArrayList<>();
     private String id;
     private double qualityLevel;
     private MetricDetails optionalMetricDetails;
@@ -52,7 +50,6 @@ public class EvalQueryQuality implements ToXContent, Writeable {
 
     public EvalQueryQuality(StreamInput in) throws IOException {
         this(in.readString(), in.readDouble());
-        this.unknownDocs = in.readList(DocumentKey::new);
         this.hits = in.readList(RatedSearchHit::new);
         this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class);
     }
@@ -61,7 +58,6 @@ public class EvalQueryQuality implements ToXContent, Writeable {
     public void writeTo(StreamOutput out) throws IOException {
         out.writeString(id);
         out.writeDouble(qualityLevel);
-        out.writeList(unknownDocs);
         out.writeList(hits);
         out.writeOptionalNamedWriteable(this.optionalMetricDetails);
     }
@@ -74,14 +70,6 @@ public class EvalQueryQuality implements ToXContent, Writeable {
         return qualityLevel;
     }
 
-    public void setUnknownDocs(List<DocumentKey> unknownDocs) {
-        this.unknownDocs = unknownDocs;
-    }
-
-    public List<DocumentKey> getUnknownDocs() {
-        return Collections.unmodifiableList(this.unknownDocs);
-    }
-
     public void addMetricDetails(MetricDetails breakdown) {
         this.optionalMetricDetails = breakdown;
     }
@@ -103,7 +91,7 @@ public class EvalQueryQuality implements ToXContent, Writeable {
         builder.startObject(id);
         builder.field("quality_level", this.qualityLevel);
         builder.startArray("unknown_docs");
-        for (DocumentKey key : unknownDocs) {
+        for (DocumentKey key : RankedListQualityMetric.filterUnknownDocuments(hits)) {
             key.toXContent(builder, params);
         }
         builder.endArray();
@@ -132,13 +120,12 @@ public class EvalQueryQuality implements ToXContent, Writeable {
         EvalQueryQuality other = (EvalQueryQuality) obj;
         return Objects.equals(id, other.id) &&
                 Objects.equals(qualityLevel, other.qualityLevel) &&
-                Objects.equals(unknownDocs, other.unknownDocs) &&
                 Objects.equals(hits, other.hits) &&
                 Objects.equals(optionalMetricDetails, other.optionalMetricDetails);
     }
 
     @Override
     public final int hashCode() {
-        return Objects.hash(id, qualityLevel, unknownDocs, hits, optionalMetricDetails);
+        return Objects.hash(id, qualityLevel, hits, optionalMetricDetails);
     }
 }

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java

@@ -35,7 +35,6 @@ import java.util.Optional;
 
 import javax.naming.directory.SearchResult;
 
-import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments;
 import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings;
 
 /**
@@ -140,7 +139,6 @@ public class PrecisionAtN implements RankedListQualityMetric {
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision);
         evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad));
         evalQueryQuality.addHitsAndRatings(ratedSearchHits);
-        evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedSearchHits));
         return evalQueryQuality;
     }
 

modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java

@@ -35,7 +35,6 @@ import java.util.Optional;
 
 import javax.naming.directory.SearchResult;
 
-import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments;
 import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsWithRatings;
 
 /**
@@ -133,7 +132,6 @@ public class ReciprocalRank implements RankedListQualityMetric {
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank);
         evalQueryQuality.addMetricDetails(new Breakdown(firstRelevant));
         evalQueryQuality.addHitsAndRatings(ratedHits);
-        evalQueryQuality.setUnknownDocs(filterUnknownDocuments(ratedHits));
         return evalQueryQuality;
     }
 

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java

@@ -30,10 +30,13 @@ import org.elasticsearch.test.ESTestCase;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 
+import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments;
+
 public class DiscountedCumulativeGainAtTests extends ESTestCase {
 
     /**
@@ -110,7 +113,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase {
         DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6);
         EvalQueryQuality result = dcg.evaluate("id", hits, rated);
         assertEquals(12.779642067948913, result.getQualityLevel(), 0.00001);
-        assertEquals(2, result.getUnknownDocs().size());
+        assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size());
 
         /**
          * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order
@@ -148,7 +151,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase {
      */
     public void testDCGAtFourMoreRatings() throws IOException, InterruptedException, ExecutionException {
         List<RatedDocument> rated = new ArrayList<>();
-        Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1};
+        Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null};
         InternalSearchHit[] hits = new InternalSearchHit[6];
         for (int i = 0; i < 6; i++) {
             if (i < relevanceRatings.length) {
@@ -160,9 +163,9 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase {
             hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0)));
         }
         DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(4);
-        EvalQueryQuality result = dcg.evaluate("id", hits, rated);
+        EvalQueryQuality result = dcg.evaluate("id",  Arrays.copyOfRange(hits, 0, 4), rated);
         assertEquals(12.392789260714371 , result.getQualityLevel(), 0.00001);
-        assertEquals(1, result.getUnknownDocs().size());
+        assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size());
 
         /**
          * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java

@@ -46,7 +46,6 @@ public class EvalQueryQualityTests extends ESTestCase {
             // TODO randomize this
             evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(1, 5));
         }
-        evalQueryQuality.setUnknownDocs(unknownDocs);
         evalQueryQuality.addHitsAndRatings(ratedHits);
         return evalQueryQuality;
     }
@@ -68,9 +67,8 @@ public class EvalQueryQualityTests extends ESTestCase {
     private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) {
         String id = original.getId();
         double qualityLevel = original.getQualityLevel();
-        List<DocumentKey> unknownDocs = new ArrayList<>(original.getUnknownDocs());
         List<RatedSearchHit> ratedHits = new ArrayList<>(original.getHitsAndRatings());
-        MetricDetails breakdown = original.getMetricDetails();
+        MetricDetails metricDetails = original.getMetricDetails();
         switch (randomIntBetween(0, 3)) {
         case 0:
             id = id + "_";
@@ -79,24 +77,20 @@ public class EvalQueryQualityTests extends ESTestCase {
             qualityLevel = qualityLevel + 0.1;
             break;
         case 2:
-            unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey());
+            if (metricDetails == null) {
-            break;
+                metricDetails = new PrecisionAtN.Breakdown(1, 5);
-        case 3:
-            if (breakdown == null) {
-                breakdown = new PrecisionAtN.Breakdown(1, 5);
             } else {
-                breakdown = null;
+                metricDetails = null;
             }
             break;
-        case 4:
+        case 3:
             ratedHits.add(RatedSearchHitTests.randomRatedSearchHit());
             break;
         default:
-            throw new IllegalStateException("The test should only allow five parameters mutated");
+            throw new IllegalStateException("The test should only allow four parameters mutated");
         }
         EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel);
-        evalQueryQuality.setUnknownDocs(unknownDocs);
+        evalQueryQuality.addMetricDetails(metricDetails);
-        evalQueryQuality.addMetricDetails(breakdown);
         evalQueryQuality.addHitsAndRatings(ratedHits);
         return evalQueryQuality;
     }

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java

@@ -36,6 +36,9 @@ import java.util.List;
 import java.util.Map.Entry;
 import java.util.Set;
 
+import static org.elasticsearch.index.rankeval.RankedListQualityMetric.filterUnknownDocuments;
+
+
 public class RankEvalRequestTests  extends ESIntegTestCase {
     @Override
     protected Collection<Class<? extends Plugin>> transportClientPlugins() {
@@ -93,7 +96,7 @@ public class RankEvalRequestTests  extends ESIntegTestCase {
         for (Entry<String, EvalQueryQuality> entry : entrySet) {
             EvalQueryQuality quality = entry.getValue();
             if (entry.getKey() == "amsterdam_query") {
-                assertEquals(2, quality.getUnknownDocs().size());
+                assertEquals(2, filterUnknownDocuments(quality.getHitsAndRatings()).size());
                 List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
                 assertEquals(6, hitsAndRatings.size());
                 for (RatedSearchHit hit : hitsAndRatings) {
@@ -106,7 +109,7 @@ public class RankEvalRequestTests  extends ESIntegTestCase {
                 }
             }
             if (entry.getKey() == "berlin_query") {
-                assertEquals(5, quality.getUnknownDocs().size());
+                assertEquals(5, filterUnknownDocuments(quality.getHitsAndRatings()).size());
                 List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
                 assertEquals(6, hitsAndRatings.size());
                 for (RatedSearchHit hit : hitsAndRatings) {

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java

@@ -46,7 +46,6 @@ public class RankEvalResponseTests extends ESTestCase {
                 unknownDocs.add(DocumentKeyTests.createRandomRatedDocumentKey());
             }
             EvalQueryQuality evalQuality = new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true));
-            evalQuality.setUnknownDocs(unknownDocs);
             partials.put(id, evalQuality);
         }
         return new RankEvalResponse(randomDouble(), partials);

modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java

@@ -46,7 +46,7 @@ public class ReciprocalRankTests extends ESTestCase {
         reciprocalRank.setMaxAcceptableRank(maxRank);
         assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank());
 
-        SearchHit[] hits = toSearchHits(0, 9, "test", "type");
+        SearchHit[] hits = createSearchHits(0, 9, "test", "type");
         List<RatedDocument> ratedDocs = new ArrayList<>();
         int relevantAt = 5;
         for (int i = 0; i < 10; i++) {
@@ -76,7 +76,7 @@ public class ReciprocalRankTests extends ESTestCase {
 
     public void testEvaluationOneRelevantInResults() {
         ReciprocalRank reciprocalRank = new ReciprocalRank();
-        SearchHit[] hits = toSearchHits(0, 9, "test", "type");
+        SearchHit[] hits = createSearchHits(0, 9, "test", "type");
         List<RatedDocument> ratedDocs = new ArrayList<>();
         // mark one of the ten docs relevant
         int relevantAt = randomIntBetween(0, 9);
@@ -105,7 +105,7 @@ public class ReciprocalRankTests extends ESTestCase {
         rated.add(new RatedDocument("test", "testtype", "2", 2));
         rated.add(new RatedDocument("test", "testtype", "3", 3));
         rated.add(new RatedDocument("test", "testtype", "4", 4));
-        SearchHit[] hits = toSearchHits(0, 5, "test", "testtype");
+        SearchHit[] hits = createSearchHits(0, 5, "test", "testtype");
 
         ReciprocalRank reciprocalRank = new ReciprocalRank();
         reciprocalRank.setRelevantRatingThreshhold(2);
@@ -125,7 +125,7 @@ public class ReciprocalRankTests extends ESTestCase {
 
     public void testEvaluationNoRelevantInResults() {
         ReciprocalRank reciprocalRank = new ReciprocalRank();
-        SearchHit[] hits = toSearchHits(0, 9, "test", "type");
+        SearchHit[] hits = createSearchHits(0, 9, "test", "type");
         List<RatedDocument> ratedDocs = new ArrayList<>();
         EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
         assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
@@ -144,9 +144,13 @@ public class ReciprocalRankTests extends ESTestCase {
         assertEquals(testItem.hashCode(), parsedItem.hashCode());
     }
 
-    private static SearchHit[] toSearchHits(int from, int to, String index, String type) {
+    /**
-        InternalSearchHit[] hits = new InternalSearchHit[to - from];
+     * Create InternalSearchHits for testing, starting from dociId 'from' up to docId 'to'.
-        for (int i = from; i < to; i++) {
+     * The search hits index and type also need to be provided
+     */
+    private static SearchHit[] createSearchHits(int from, int to, String index, String type) {
+        InternalSearchHit[] hits = new InternalSearchHit[to + 1 - from];
+        for (int i = from; i <= to; i++) {
             hits[i] = new InternalSearchHit(i, i+"", new Text(type), Collections.emptyMap());
             hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0));
         }