From 29402a28e0d1d8bfad13ec8e5e4dc58b7be36b63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 22 Sep 2016 12:17:10 +0200 Subject: [PATCH] RankEval: Adding details section to response (#20497) In order to understand how well particular queries in a joint ranking evaluation request work we want to break down the overall metric into its components, each contributed by a particular query. The response structure now has a `details` section under which we can summarize this information. Each sub-section is keyed by the query-id and currently only contains the partial metric and the unknown_docs section for each query. --- .../rankeval/DiscountedCumulativeGainAt.java | 25 +++-- .../index/rankeval/EvalQueryQuality.java | 102 +++++++++++++++-- .../index/rankeval/MetricDetails.java | 27 +++++ .../index/rankeval/PrecisionAtN.java | 72 +++++++++++- .../index/rankeval/RankEvalPlugin.java | 14 ++- .../index/rankeval/RankEvalResponse.java | 58 ++++------ .../rankeval/RankedListQualityMetric.java | 4 +- .../index/rankeval/RatedDocumentKey.java | 14 ++- .../index/rankeval/ReciprocalRank.java | 60 +++++++++- .../rankeval/TransportRankEvalAction.java | 8 +- .../DiscountedCumulativeGainAtTests.java | 39 +++++-- .../index/rankeval/EvalQueryQualityTests.java | 105 ++++++++++++++++++ .../index/rankeval/PrecisionAtNTests.java | 33 ++++-- .../index/rankeval/RankEvalRequestTests.java | 8 +- .../index/rankeval/RankEvalResponseTests.java | 16 +-- .../index/rankeval/RankEvalSpecTests.java | 2 +- ...estHelper.java => RankEvalTestHelper.java} | 27 ++++- .../index/rankeval/RatedDocumentKeyTests.java | 40 ++++--- .../index/rankeval/RatedDocumentTests.java | 6 +- .../index/rankeval/RatedRequestsTests.java | 4 +- .../index/rankeval/ReciprocalRankTests.java | 24 ++-- .../test/rank_eval/10_basic.yaml | 28 ++++- .../rest-api-spec/test/rank_eval/20_dcg.yaml | 8 ++ 23 files changed, 571 insertions(+), 153 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{XContentTestHelper.java => RankEvalTestHelper.java} (57%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index 36d2a208353..b0a806287ca 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -30,8 +30,8 @@ import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; +import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -139,13 +139,13 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { } @Override - public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { Map ratedDocsByKey = new HashMap<>(); for (RatedDocument doc : ratedDocs) { ratedDocsByKey.put(doc.getKey(), doc); } - Collection unknownDocIds = new ArrayList<>(); + List unknownDocIds = new ArrayList<>(); List ratings = new ArrayList<>(); for (int i = 0; (i < position && i < hits.length); i++) { RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); @@ -156,24 +156,29 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { unknownDocIds.add(id); if (unknownDocRating != null) { ratings.add(unknownDocRating); + } else { + // we add null here so that the later computation knows this position had no rating + ratings.add(null); } } } double dcg = computeDCG(ratings); if (normalize) { - Collections.sort(ratings, Collections.reverseOrder()); + Collections.sort(ratings, Comparator.nullsLast(Collections.reverseOrder())); double idcg = computeDCG(ratings); dcg = dcg / idcg; } - return new EvalQueryQuality(dcg, unknownDocIds); + return new EvalQueryQuality(taskId, dcg, unknownDocIds); } private static double computeDCG(List ratings) { int rank = 1; double dcg = 0; - for (int rating : ratings) { - dcg += (Math.pow(2, rating) - 1) / ((Math.log(rank + 1) / LOG2)); + for (Integer rating : ratings) { + if (rating != null) { + dcg += (Math.pow(2, rating) - 1) / ((Math.log(rank + 1) / LOG2)); + } rank++; } return dcg; @@ -208,7 +213,7 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { builder.endObject(); return builder; } - + @Override public final boolean equals(Object obj) { if (this == obj) { @@ -222,9 +227,11 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { Objects.equals(normalize, other.normalize) && Objects.equals(unknownDocRating, other.unknownDocRating); } - + @Override public final int hashCode() { return Objects.hash(position, normalize, unknownDocRating); } + + // TODO maybe also add debugging breakdown here } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index 9f8a192d6e7..58268522cc1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -19,29 +19,107 @@ package org.elasticsearch.index.rankeval; -import java.util.Collection; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Objects;; /** - * Returned for each search specification. Summarizes the measured quality - * metric for this search request and adds the document ids found that were in - * the search result but not annotated in the original request. + * This class represents the partial information from running the ranking evaluation metric on one + * request alone. It contains all information necessary to render the response for this part of the + * overall evaluation. */ -public class EvalQueryQuality { +public class EvalQueryQuality implements ToXContent, Writeable { + + /** documents seen as result for one request that were not annotated.*/ + private List unknownDocs; + private String id; private double qualityLevel; + private MetricDetails optionalMetricDetails; - private Collection unknownDocs; - - public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { - this.qualityLevel = qualityLevel; - this.unknownDocs = unknownDocs; + public EvalQueryQuality(String id, double qualityLevel, List unknownDocs) { + this.id = id; + this.unknownDocs = unknownDocs; + this.qualityLevel = qualityLevel; } - public Collection getUnknownDocs() { - return unknownDocs; + public EvalQueryQuality(StreamInput in) throws IOException { + this(in.readString(), in.readDouble(), in.readList(RatedDocumentKey::new)); + this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class); + } + + public String getId() { + return id; } public double getQualityLevel() { return qualityLevel; } + public List getUnknownDocs() { + return Collections.unmodifiableList(this.unknownDocs); + } + + public void addMetricDetails(MetricDetails breakdown) { + this.optionalMetricDetails = breakdown; + } + + public MetricDetails getMetricDetails() { + return this.optionalMetricDetails; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(id); + out.writeDouble(qualityLevel); + out.writeVInt(unknownDocs.size()); + for (RatedDocumentKey key : unknownDocs) { + key.writeTo(out); + } + out.writeOptionalNamedWriteable(this.optionalMetricDetails); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(id); + builder.field("quality_level", this.qualityLevel); + builder.startArray("unknown_docs"); + for (RatedDocumentKey key : unknownDocs) { + key.toXContent(builder, params); + } + builder.endArray(); + if (optionalMetricDetails != null) { + builder.startObject("metric_details"); + optionalMetricDetails.toXContent(builder, params); + builder.endObject(); + } + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + EvalQueryQuality other = (EvalQueryQuality) obj; + return Objects.equals(id, other.id) && + Objects.equals(qualityLevel, other.qualityLevel) && + Objects.equals(unknownDocs, other.unknownDocs) && + Objects.equals(optionalMetricDetails, other.optionalMetricDetails); + } + + @Override + public final int hashCode() { + return Objects.hash(id, qualityLevel, unknownDocs, optionalMetricDetails); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java new file mode 100644 index 00000000000..af838111427 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.NamedWriteable; +import org.elasticsearch.common.xcontent.ToXContent; + +public interface MetricDetails extends ToXContent, NamedWriteable { + +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index ac8675a2a37..83b9de04f5d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -120,7 +120,7 @@ public class PrecisionAtN extends RankedListQualityMetric { * @return precision at n for above {@link SearchResult} list. **/ @Override - public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { Collection relevantDocIds = new ArrayList<>(); Collection irrelevantDocIds = new ArrayList<>(); @@ -134,7 +134,7 @@ public class PrecisionAtN extends RankedListQualityMetric { int good = 0; int bad = 0; - Collection unknownDocIds = new ArrayList<>(); + List unknownDocIds = new ArrayList<>(); for (int i = 0; (i < n && i < hits.length); i++) { RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); if (relevantDocIds.contains(hitKey)) { @@ -146,7 +146,9 @@ public class PrecisionAtN extends RankedListQualityMetric { } } double precision = (double) good / (good + bad); - return new EvalQueryQuality(precision, unknownDocIds); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision, unknownDocIds); + evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad)); + return evalQueryQuality; } // TODO add abstraction that also works for other metrics @@ -194,9 +196,71 @@ public class PrecisionAtN extends RankedListQualityMetric { PrecisionAtN other = (PrecisionAtN) obj; return Objects.equals(n, other.n); } - + @Override public final int hashCode() { return Objects.hash(n); } + + public static class Breakdown implements MetricDetails { + + public static final String DOCS_RETRIEVED_FIELD = "docs_retrieved"; + public static final String RELEVANT_DOCS_RETRIEVED_FIELD = "relevant_docs_retrieved"; + private int relevantRetrieved; + private int retrieved; + + public Breakdown(int relevantRetrieved, int retrieved) { + this.relevantRetrieved = relevantRetrieved; + this.retrieved = retrieved; + } + + public Breakdown(StreamInput in) throws IOException { + this.relevantRetrieved = in.readVInt(); + this.retrieved = in.readVInt(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(RELEVANT_DOCS_RETRIEVED_FIELD, relevantRetrieved); + builder.field(DOCS_RETRIEVED_FIELD, retrieved); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(relevantRetrieved); + out.writeVInt(retrieved); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public int getRelevantRetrieved() { + return relevantRetrieved; + } + + public int getRetrieved() { + return retrieved; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + PrecisionAtN.Breakdown other = (PrecisionAtN.Breakdown) obj; + return Objects.equals(relevantRetrieved, other.relevantRetrieved) && + Objects.equals(retrieved, other.retrieved); + } + + @Override + public final int hashCode() { + return Objects.hash(relevantRetrieved, retrieved); + } + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java index bb59ff6051f..c741f59f74c 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java @@ -49,9 +49,15 @@ public class RankEvalPlugin extends Plugin implements ActionPlugin { */ @Override public List getNamedWriteables() { - List metrics = new ArrayList<>(); - metrics.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new)); - metrics.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); - return metrics; + List namedWriteables = new ArrayList<>(); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGainAt.NAME, + DiscountedCumulativeGainAt::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, PrecisionAtN.NAME, + PrecisionAtN.Breakdown::new)); + namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, ReciprocalRank.NAME, + ReciprocalRank.Breakdown::new)); + return namedWriteables; } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 43b66b46861..81994e86f24 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -26,8 +26,7 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Objects; @@ -47,41 +46,37 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { /**Average precision observed when issuing query intents with this specification.*/ private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ - private Map> unknownDocs; + private Map details; public RankEvalResponse() { } - public RankEvalResponse(double qualityLevel, Map> unknownDocs) { + public RankEvalResponse(double qualityLevel, Map partialResults) { this.qualityLevel = qualityLevel; - this.unknownDocs = unknownDocs; + this.details = partialResults; } public double getQualityLevel() { return qualityLevel; } - public Map> getUnknownDocs() { - return unknownDocs; + public Map getPartialResults() { + return Collections.unmodifiableMap(details); } @Override public String toString() { - return "RankEvalResponse, quality: " + qualityLevel + ", unknown docs: " + unknownDocs; + return "RankEvalResponse, quality: " + qualityLevel + ", partial results: " + details; } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeDouble(qualityLevel); - out.writeVInt(unknownDocs.size()); - for (String queryId : unknownDocs.keySet()) { + out.writeVInt(details.size()); + for (String queryId : details.keySet()) { out.writeString(queryId); - Collection collection = unknownDocs.get(queryId); - out.writeVInt(collection.size()); - for (RatedDocumentKey key : collection) { - key.writeTo(out); - } + details.get(queryId).writeTo(out); } } @@ -89,16 +84,12 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { public void readFrom(StreamInput in) throws IOException { super.readFrom(in); this.qualityLevel = in.readDouble(); - int unknownDocumentSets = in.readVInt(); - this.unknownDocs = new HashMap<>(unknownDocumentSets); - for (int i = 0; i < unknownDocumentSets; i++) { + int partialResultSize = in.readVInt(); + this.details = new HashMap<>(partialResultSize); + for (int i = 0; i < partialResultSize; i++) { String queryId = in.readString(); - int numberUnknownDocs = in.readVInt(); - Collection collection = new ArrayList<>(numberUnknownDocs); - for (int d = 0; d < numberUnknownDocs; d++) { - collection.add(new RatedDocumentKey(in)); - } - this.unknownDocs.put(queryId, collection); + EvalQueryQuality partial = new EvalQueryQuality(in); + this.details.put(queryId, partial); } } @@ -106,18 +97,9 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject("rank_eval"); builder.field("quality_level", qualityLevel); - builder.startObject("unknown_docs"); - for (String key : unknownDocs.keySet()) { - Collection keys = unknownDocs.get(key); - builder.startArray(key); - for (RatedDocumentKey docKey : keys) { - builder.startObject(); - builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), docKey.getIndex()); - builder.field(RatedDocument.TYPE_FIELD.getPreferredName(), docKey.getType()); - builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docKey.getDocID()); - builder.endObject(); - } - builder.endArray(); + builder.startObject("details"); + for (String key : details.keySet()) { + details.get(key).toXContent(builder, params); } builder.endObject(); builder.endObject(); @@ -134,11 +116,11 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { } RankEvalResponse other = (RankEvalResponse) obj; return Objects.equals(qualityLevel, other.qualityLevel) && - Objects.equals(unknownDocs, other.unknownDocs); + Objects.equals(details, other.details); } @Override public final int hashCode() { - return Objects.hash(getClass(), qualityLevel, unknownDocs); + return Objects.hash(qualityLevel, details); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index e423ab3533c..a34cbd058cd 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -44,10 +44,12 @@ public abstract class RankedListQualityMetric extends ToXContentToBytes implemen * Returns a single metric representing the ranking quality of a set of returned documents * wrt. to a set of document Ids labeled as relevant for this search. * + * @param taskId the id of the query for which the ranking is currently evaluated * @param hits the result hits as returned by some search + * @param ratedDocs the documents that were ranked by human annotators for this query case * @return some metric representing the quality of the result hit list wrt. to relevant doc ids. * */ - public abstract EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs); + public abstract EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs); public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { RankedListQualityMetric rc; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index ee08cf018da..35da907189f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -19,14 +19,16 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentBuilder; import java.io.IOException; import java.util.Objects; -public class RatedDocumentKey implements Writeable { +public class RatedDocumentKey extends ToXContentToBytes implements Writeable { private String docId; private String type; @@ -93,4 +95,14 @@ public class RatedDocumentKey implements Writeable { public final int hashCode() { return Objects.hash(index, type, docId); } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), index); + builder.field(RatedDocument.TYPE_FIELD.getPreferredName(), type); + builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docId); + builder.endObject(); + return builder; + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 803900a3521..c249706a948 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -30,7 +30,6 @@ import org.elasticsearch.search.SearchHit; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.List; import java.util.Objects; @@ -114,7 +113,7 @@ public class ReciprocalRank extends RankedListQualityMetric { * @return reciprocal Rank for above {@link SearchResult} list. **/ @Override - public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { + public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List ratedDocs) { Set relevantDocIds = new HashSet<>(); Set irrelevantDocIds = new HashSet<>(); for (RatedDocument doc : ratedDocs) { @@ -125,7 +124,7 @@ public class ReciprocalRank extends RankedListQualityMetric { } } - Collection unknownDocIds = new ArrayList<>(); + List unknownDocIds = new ArrayList<>(); int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { @@ -141,7 +140,9 @@ public class ReciprocalRank extends RankedListQualityMetric { } double reciprocalRank = (firstRelevant == -1) ? 0 : 1.0d / firstRelevant; - return new EvalQueryQuality(reciprocalRank, unknownDocIds); + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank, unknownDocIds); + evalQueryQuality.addMetricDetails(new Breakdown(firstRelevant)); + return evalQueryQuality; } @Override @@ -184,9 +185,58 @@ public class ReciprocalRank extends RankedListQualityMetric { ReciprocalRank other = (ReciprocalRank) obj; return Objects.equals(maxAcceptableRank, other.maxAcceptableRank); } - + @Override public final int hashCode() { return Objects.hash(maxAcceptableRank); } + + public static class Breakdown implements MetricDetails { + + private int firstRelevantRank; + + public Breakdown(int firstRelevantRank) { + this.firstRelevantRank = firstRelevantRank; + } + + public Breakdown(StreamInput in) throws IOException { + this.firstRelevantRank = in.readVInt(); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field("first_relevant", firstRelevantRank); + return builder; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(firstRelevantRank); + } + + @Override + public String getWriteableName() { + return NAME; + } + + public int getFirstRelevantRank() { + return firstRelevantRank; + } + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + ReciprocalRank.Breakdown other = (ReciprocalRank.Breakdown) obj; + return Objects.equals(firstRelevantRank, other.firstRelevantRank); + } + + @Override + public final int hashCode() { + return Objects.hash(firstRelevantRank); + } + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index f7f21eb81e7..65dbbc0ecc9 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -88,7 +88,6 @@ public class TransportRankEvalAction extends HandledTransportAction partialResults; private RankEvalSpec task; - private Map> unknownDocs; private AtomicInteger responseCounter; public RankEvalActionListener(ActionListener listener, RankEvalSpec task, RatedRequest specification, @@ -98,21 +97,20 @@ public class TransportRankEvalAction extends HandledTransportAction rated = new ArrayList<>(); - int[] relevanceRatings = new int[] { 3, 2, 3}; + Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1}; InternalSearchHit[] hits = new InternalSearchHit[6]; for (int i = 0; i < 6; i++) { if (i < relevanceRatings.length) { - rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + if (relevanceRatings[i] != null) { + rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i])); + } } hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new ShardId("index", "uuid", 0))); } DiscountedCumulativeGainAt dcg = new DiscountedCumulativeGainAt(6); - EvalQueryQuality result = dcg.evaluate(hits, rated); - assertEquals(12.392789260714371, result.getQualityLevel(), 0.00001); - assertEquals(3, result.getUnknownDocs().size()); + EvalQueryQuality result = dcg.evaluate("id", hits, rated); + assertEquals(12.779642067948913, result.getQualityLevel(), 0.00001); + assertEquals(2, result.getUnknownDocs().size()); + + /** + * Check with normalization: to get the maximal possible dcg, sort documents by relevance in descending order + * + * rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1) + * ------------------------------------------------------------------------------------------- + * 1 | 3 | 7.0 | 1.0  | 7.0 + * 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202 + * 3 | 2 | 3.0 | 2.0  | 1.5 + * 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339 + * 5 | n.a | n.a | n.a.  | n.a. + * 6 | n.a | n.a | n.a  | n.a + * + * idcg = 13.347184833073591 (sum of last column) + */ + dcg.setNormalize(true); + assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001); } public void testParseFromXContent() throws IOException { @@ -131,7 +150,7 @@ public class DiscountedCumulativeGainAtTests extends ESTestCase { } public void testXContentRoundtrip() throws IOException { DiscountedCumulativeGainAt testItem = createTestItem(); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); DiscountedCumulativeGainAt parsedItem = DiscountedCumulativeGainAt.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java new file mode 100644 index 00000000000..8924ff0f66d --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java @@ -0,0 +1,105 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.io.stream.BytesStreamOutput; +import org.elasticsearch.common.io.stream.NamedWriteableAwareStreamInput; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class EvalQueryQualityTests extends ESTestCase { + + private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry(new RankEvalPlugin().getNamedWriteables()); + + public static EvalQueryQuality randomEvalQueryQuality() { + List unknownDocs = new ArrayList<>(); + int numberOfUnknownDocs = randomInt(5); + for (int i = 0; i < numberOfUnknownDocs; i++) { + unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); + } + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAsciiOfLength(10), randomDoubleBetween(0.0, 1.0, true), unknownDocs); + if (randomBoolean()) { + // TODO randomize this + evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(1, 5)); + } + return evalQueryQuality; + } + + private static EvalQueryQuality copy(EvalQueryQuality original) throws IOException { + try (BytesStreamOutput output = new BytesStreamOutput()) { + original.writeTo(output); + try (StreamInput in = new NamedWriteableAwareStreamInput(output.bytes().streamInput(), namedWritableRegistry)) { + return new EvalQueryQuality(in); + } + } + } + + public void testSerialization() throws IOException { + EvalQueryQuality original = randomEvalQueryQuality(); + EvalQueryQuality deserialized = copy(original); + assertEquals(deserialized, original); + assertEquals(deserialized.hashCode(), original.hashCode()); + assertNotSame(deserialized, original); + } + + public void testEqualsAndHash() throws IOException { + EvalQueryQuality testItem = randomEvalQueryQuality(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + copy(testItem)); + } + + private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) { + String id = original.getId(); + double qualityLevel = original.getQualityLevel(); + List unknownDocs = original.getUnknownDocs(); + MetricDetails breakdown = original.getMetricDetails(); + switch (randomIntBetween(0, 3)) { + case 0: + id = id + "_"; + break; + case 1: + qualityLevel = qualityLevel + 0.1; + break; + case 2: + unknownDocs = new ArrayList<>(unknownDocs); + unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); + break; + case 3: + if (breakdown == null) { + breakdown = new PrecisionAtN.Breakdown(1, 5); + } else { + breakdown = null; + } + break; + default: + throw new IllegalStateException("The test should only allow three parameters mutated"); + } + EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel, unknownDocs); + evalQueryQuality.addMetricDetails(breakdown); + return evalQueryQuality; + } + + +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index bc78484832f..e892126a37a 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -46,7 +46,10 @@ public class PrecisionAtNTests extends ESTestCase { InternalSearchHit[] hits = new InternalSearchHit[1]; hits[0] = new InternalSearchHit(0, "0", new Text("testtype"), Collections.emptyMap()); hits[0].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); - assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals(1, evaluated.getQualityLevel(), 0.00001); + assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { @@ -61,7 +64,10 @@ public class PrecisionAtNTests extends ESTestCase { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } - assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001); + assertEquals(4, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } /** @@ -82,7 +88,10 @@ public class PrecisionAtNTests extends ESTestCase { } PrecisionAtN precisionAtN = new PrecisionAtN(5); precisionAtN.setRelevantRatingThreshhold(2); - assertEquals((double) 3 / 5, precisionAtN.evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = precisionAtN.evaluate("id", hits, rated); + assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001); + assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { @@ -97,7 +106,10 @@ public class PrecisionAtNTests extends ESTestCase { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } - assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); + assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException { @@ -112,7 +124,10 @@ public class PrecisionAtNTests extends ESTestCase { hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } - assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", hits, rated); + assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001); + assertEquals(2, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved()); + assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved()); } public void testParseFromXContent() throws IOException { @@ -129,9 +144,9 @@ public class PrecisionAtNTests extends ESTestCase { public void testCombine() { PrecisionAtN metric = new PrecisionAtN(); Vector partialResults = new Vector<>(3); - partialResults.add(new EvalQueryQuality(0.1, emptyList())); - partialResults.add(new EvalQueryQuality(0.2, emptyList())); - partialResults.add(new EvalQueryQuality(0.6, emptyList())); + partialResults.add(new EvalQueryQuality("a", 0.1, emptyList())); + partialResults.add(new EvalQueryQuality("b", 0.2, emptyList())); + partialResults.add(new EvalQueryQuality("c", 0.6, emptyList())); assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } @@ -142,7 +157,7 @@ public class PrecisionAtNTests extends ESTestCase { public void testXContentRoundtrip() throws IOException { PrecisionAtN testItem = createTestItem(); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); PrecisionAtN parsedItem = PrecisionAtN.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index 9fee4685e6c..b6852be8311 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -83,14 +83,14 @@ public class RankEvalRequestTests extends ESIntegTestCase { RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet(); assertEquals(1.0, response.getQualityLevel(), Double.MIN_VALUE); - Set>> entrySet = response.getUnknownDocs().entrySet(); + Set> entrySet = response.getPartialResults().entrySet(); assertEquals(2, entrySet.size()); - for (Entry> entry : entrySet) { + for (Entry entry : entrySet) { if (entry.getKey() == "amsterdam_query") { - assertEquals(2, entry.getValue().size()); + assertEquals(2, entry.getValue().getUnknownDocs().size()); } if (entry.getKey() == "berlin_query") { - assertEquals(5, entry.getValue().size()); + assertEquals(5, entry.getValue().getUnknownDocs().size()); } } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java index b5b68ec22ae..73ba2301d58 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java @@ -29,7 +29,6 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -37,17 +36,18 @@ import java.util.Map; public class RankEvalResponseTests extends ESTestCase { private static RankEvalResponse createRandomResponse() { - Map> unknownDocs = new HashMap<>(); - int numberOfSets = randomIntBetween(0, 5); - for (int i = 0; i < numberOfSets; i++) { - List ids = new ArrayList<>(); + int numberOfRequests = randomIntBetween(0, 5); + Map partials = new HashMap<>(numberOfRequests); + for (int i = 0; i < numberOfRequests; i++) { + String id = randomAsciiOfLengthBetween(3, 10); int numberOfUnknownDocs = randomIntBetween(0, 5); + List unknownDocs = new ArrayList<>(numberOfUnknownDocs); for (int d = 0; d < numberOfUnknownDocs; d++) { - ids.add(new RatedDocumentKey(randomAsciiOfLength(5), randomAsciiOfLength(5), randomAsciiOfLength(5))); + unknownDocs.add(RatedDocumentKeyTests.createRandomRatedDocumentKey()); } - unknownDocs.put(randomAsciiOfLength(5), ids); + partials.put(id, new EvalQueryQuality(id, randomDoubleBetween(0.0, 1.0, true), unknownDocs)); } - return new RankEvalResponse(randomDouble(), unknownDocs ); + return new RankEvalResponse(randomDouble(), partials); } public void testSerialization() throws IOException { diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java index 97fd0a9f281..59c17d71d32 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -89,7 +89,7 @@ public class RankEvalSpecTests extends ESTestCase { RankEvalSpec testItem = new RankEvalSpec(specs, metric); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java similarity index 57% rename from modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java rename to modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java index e02772ad6f6..4218ae5d5cd 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentTestHelper.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalTestHelper.java @@ -30,9 +30,16 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -public class XContentTestHelper { +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.not; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; - public static XContentParser roundtrip(ToXContentToBytes testItem) throws IOException { +public class RankEvalTestHelper { + + public static XContentParser roundtrip(ToXContentToBytes testItem) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(ESTestCase.randomFrom(XContentType.values())); if (ESTestCase.randomBoolean()) { builder.prettyPrint(); @@ -42,4 +49,20 @@ public class XContentTestHelper { XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); return itemParser; } + + public static void testHashCodeAndEquals(Object testItem, Object mutation, Object secondCopy) { + assertFalse("testItem is equal to null", testItem.equals(null)); + assertFalse("testItem is equal to incompatible type", testItem.equals("")); + assertTrue("testItem is not equal to self", testItem.equals(testItem)); + assertThat("same testItem's hashcode returns different values if called multiple times", testItem.hashCode(), + equalTo(testItem.hashCode())); + + assertThat("different testItem should not be equal", mutation, not(equalTo(testItem))); + + assertNotSame("testItem copy is not same as original", testItem, secondCopy); + assertTrue("testItem is not equal to its copy", testItem.equals(secondCopy)); + assertTrue("equals is not symmetric", secondCopy.equals(testItem)); + assertThat("testItem copy's hashcode is different from original hashcode", secondCopy.hashCode(), + equalTo(testItem.hashCode())); + } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java index 26661c086a6..231a1b8e586 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -23,45 +23,43 @@ import org.elasticsearch.test.ESTestCase; import java.io.IOException; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.not; public class RatedDocumentKeyTests extends ESTestCase { - public void testEqualsAndHash() throws IOException { + static RatedDocumentKey createRandomRatedDocumentKey() { String index = randomAsciiOfLengthBetween(0, 10); String type = randomAsciiOfLengthBetween(0, 10); String docId = randomAsciiOfLengthBetween(0, 10); + return new RatedDocumentKey(index, type, docId); + } - RatedDocumentKey testItem = new RatedDocumentKey(index, type, docId); + public RatedDocumentKey createRandomTestItem() { + return createRandomRatedDocumentKey(); + } - assertFalse("key is equal to null", testItem.equals(null)); - assertFalse("key is equal to incompatible type", testItem.equals("")); - assertTrue("key is not equal to self", testItem.equals(testItem)); - assertThat("same key's hashcode returns different values if called multiple times", testItem.hashCode(), - equalTo(testItem.hashCode())); - - RatedDocumentKey mutation; + public RatedDocumentKey mutateTestItem(RatedDocumentKey original) { + String index = original.getIndex(); + String type = original.getType(); + String docId = original.getDocID(); switch (randomIntBetween(0, 2)) { case 0: - mutation = new RatedDocumentKey(testItem.getIndex() + "_foo", testItem.getType(), testItem.getDocID()); + index = index + "_"; break; case 1: - mutation = new RatedDocumentKey(testItem.getIndex(), testItem.getType() + "_foo", testItem.getDocID()); + type = type + "_"; break; case 2: - mutation = new RatedDocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID() + "_foo"); + docId = docId + "_"; break; default: throw new IllegalStateException("The test should only allow three parameters mutated"); } + return new RatedDocumentKey(index, type, docId); + } - assertThat("different keys should not be equal", mutation, not(equalTo(testItem))); - - RatedDocumentKey secondEqualKey = new RatedDocumentKey(index, type, docId); - assertTrue("key is not equal to its copy", testItem.equals(secondEqualKey)); - assertTrue("equals is not symmetric", secondEqualKey.equals(testItem)); - assertThat("key copy's hashcode is different from original hashcode", secondEqualKey.hashCode(), - equalTo(testItem.hashCode())); + public void testEqualsAndHash() throws IOException { + RatedDocumentKey testItem = createRandomRatedDocumentKey(); + RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), + new RatedDocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID())); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index b3af34423c7..33266b897f7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -27,7 +27,7 @@ import java.io.IOException; public class RatedDocumentTests extends ESTestCase { - public static RatedDocument createTestItem() { + public static RatedDocument createRatedDocument() { String index = randomAsciiOfLength(10); String type = randomAsciiOfLength(10); String docId = randomAsciiOfLength(10); @@ -37,8 +37,8 @@ public class RatedDocumentTests extends ESTestCase { } public void testXContentParsing() throws IOException { - RatedDocument testItem = createTestItem(); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + RatedDocument testItem = createRatedDocument(); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); assertNotSame(testItem, parsedItem); assertEquals(testItem, parsedItem); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 0d1615a5d05..c325181727e 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -76,7 +76,7 @@ public class RatedRequestsTests extends ESTestCase { List ratedDocs = new ArrayList<>(); int size = randomIntBetween(0, 2); for (int i = 0; i < size; i++) { - ratedDocs.add(RatedDocumentTests.createTestItem()); + ratedDocs.add(RatedDocumentTests.createRatedDocument()); } return new RatedRequest(specId, testRequest, indices, types, ratedDocs); @@ -96,7 +96,7 @@ public class RatedRequestsTests extends ESTestCase { } RatedRequest testItem = createTestItem(indices, types); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index c3f951b09b4..6dc5e670db5 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -63,17 +63,19 @@ public class ReciprocalRankTests extends ESTestCase { } int rankAtFirstRelevant = relevantAt + 1; - EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); if (rankAtFirstRelevant <= maxRank) { assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); // check that if we lower maxRank by one, we don't find any result and get 0.0 quality level reciprocalRank = new ReciprocalRank(rankAtFirstRelevant - 1); - evaluation = reciprocalRank.evaluate(hits, ratedDocs); + evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } else { assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(-1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } } @@ -95,8 +97,9 @@ public class ReciprocalRankTests extends ESTestCase { } } - EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE); + assertEquals(relevantAt + 1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } /** @@ -119,15 +122,17 @@ public class ReciprocalRankTests extends ESTestCase { ReciprocalRank reciprocalRank = new ReciprocalRank(); reciprocalRank.setRelevantRatingThreshhold(2); - assertEquals((double) 1 / 3, reciprocalRank.evaluate(hits, rated).getQualityLevel(), 0.00001); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated); + assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001); + assertEquals(3, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank()); } public void testCombine() { ReciprocalRank reciprocalRank = new ReciprocalRank(); Vector partialResults = new Vector<>(3); - partialResults.add(new EvalQueryQuality(0.5, emptyList())); - partialResults.add(new EvalQueryQuality(1.0, emptyList())); - partialResults.add(new EvalQueryQuality(0.75, emptyList())); + partialResults.add(new EvalQueryQuality("id1", 0.5, emptyList())); + partialResults.add(new EvalQueryQuality("id2", 1.0, emptyList())); + partialResults.add(new EvalQueryQuality("id3", 0.75, emptyList())); assertEquals(0.75, reciprocalRank.combine(partialResults), Double.MIN_VALUE); } @@ -139,7 +144,7 @@ public class ReciprocalRankTests extends ESTestCase { hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); - EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); + EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); } @@ -147,7 +152,7 @@ public class ReciprocalRankTests extends ESTestCase { int position = randomIntBetween(0, 1000); ReciprocalRank testItem = new ReciprocalRank(position); - XContentParser itemParser = XContentTestHelper.roundtrip(testItem); + XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); itemParser.nextToken(); itemParser.nextToken(); ReciprocalRank parsedItem = ReciprocalRank.fromXContent(itemParser, () -> ParseFieldMatcher.STRICT); @@ -155,5 +160,4 @@ public class ReciprocalRankTests extends ESTestCase { assertEquals(testItem, parsedItem); assertEquals(testItem.hashCode(), parsedItem.hashCode()); } - } diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 2abad7353e9..63944f899f1 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -60,8 +60,13 @@ } - match: {rank_eval.quality_level: 1} - - match: {rank_eval.unknown_docs.amsterdam_query: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} - - match: {rank_eval.unknown_docs.berlin_query: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: {rank_eval.details.amsterdam_query.quality_level: 1.0} + - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: {rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}} + - match: {rank_eval.details.berlin_query.quality_level: 1.0} + - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]} + - match: {rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}} + --- "Reciprocal Rank": @@ -105,7 +110,7 @@ - do: rank_eval: - body: { + body: { "requests" : [ { "id": "amsterdam_query", @@ -125,6 +130,13 @@ # average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663 - match: {rank_eval.quality_level: 0.41666666666666663} + - match: {rank_eval.details.amsterdam_query.quality_level: 0.3333333333333333} + - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": 3}} + - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, + {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} + - match: {rank_eval.details.berlin_query.quality_level: 0.5} + - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} + - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} - do: rank_eval: @@ -145,10 +157,18 @@ ], "metric" : { "reciprocal_rank": { + # the following will make the first query have a quality value of 0.0 "max_acceptable_rank" : 2 } } } # average is (0 + 1/2)/2 = 1/4 - - match: {rank_eval.quality_level: 0.25} + - match: {rank_eval.quality_level: 0.25} + - match: {rank_eval.details.amsterdam_query.quality_level: 0} + - match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": -1}} + - match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"}, + {"_index": "foo", "_type": "bar", "_id": "doc3"} ]} + - match: {rank_eval.details.berlin_query.quality_level: 0.5} + - match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}} + - match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]} diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml index 51f3393c21b..8310389c02a 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yaml @@ -64,6 +64,8 @@ } - match: {rank_eval.quality_level: 13.84826362927298} + - match: {rank_eval.details.dcg_query.quality_level: 13.84826362927298} + - match: {rank_eval.details.dcg_query.unknown_docs: [ ]} # reverse the order in which the results are returned (less relevant docs first) @@ -87,6 +89,8 @@ } - match: {rank_eval.quality_level: 10.29967439154499} + - match: {rank_eval.details.dcg_query_reverse.quality_level: 10.29967439154499} + - match: {rank_eval.details.dcg_query_reverse.unknown_docs: [ ]} # if we mix both, we should get the average @@ -121,3 +125,7 @@ } - match: {rank_eval.quality_level: 12.073969010408984} + - match: {rank_eval.details.dcg_query.quality_level: 13.84826362927298} + - match: {rank_eval.details.dcg_query.unknown_docs: [ ]} + - match: {rank_eval.details.dcg_query_reverse.quality_level: 10.29967439154499} + - match: {rank_eval.details.dcg_query_reverse.unknown_docs: [ ]}