honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rank Eval: Handle precion@ edge case 

There's a currently unhandled edge case for the precion@ metric. When none of
the search hits in the result are rated, we have neither true nor false
positives which currently leads to division by zero. We should return a precion
of 0.0 in this case.
This commit is contained in:
Christoph Büscher 2016-11-03 12:30:39 +01:00
parent 25565b9baa
commit 2dad72e68c
2 changed files with 22 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
View File

@ -122,22 +122,25 @@ public class PrecisionAtN implements RankedListQualityMetric {
**/
@Override
public EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List<RatedDocument> ratedDocs) {
int good = 0;
int bad = 0;
int truePositives = 0;
int falsePositives = 0;
List<RatedSearchHit> ratedSearchHits = joinHitsWithRatings(hits, ratedDocs);
for (RatedSearchHit hit : ratedSearchHits) {
Optional<Integer> rating = hit.getRating();
if (rating.isPresent()) {
if (rating.get() >= this.relevantRatingThreshhold) {
good++;
truePositives++;
} else {
bad++;
falsePositives++;
}
}
}
double precision = (double) good / (good + bad);
double precision = 0.0;
if (truePositives + falsePositives > 0) {
precision = (double) truePositives / (truePositives + falsePositives);
}
EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision);
evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(good, good + bad));
evalQueryQuality.addMetricDetails(new PrecisionAtN.Breakdown(truePositives, truePositives + falsePositives));
evalQueryQuality.addHitsAndRatings(ratedSearchHits);
return evalQueryQuality;
}

19
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java
View File

@ -35,11 +35,10 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Vector;
import java.util.concurrent.ExecutionException;
public class PrecisionAtNTests extends ESTestCase {
public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException {
public void testPrecisionAtFiveCalculation() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated);
@ -48,7 +47,7 @@ public class PrecisionAtNTests extends ESTestCase {
assertEquals(1, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException {
public void testPrecisionAtFiveIgnoreOneResult() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal()));
@ -65,7 +64,7 @@ public class PrecisionAtNTests extends ESTestCase {
* test that the relevant rating threshold can be set to something larger than 1.
* e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and 4 to be relevant
*/
public void testPrecisionAtFiveRelevanceThreshold() throws IOException, InterruptedException, ExecutionException {
public void testPrecisionAtFiveRelevanceThreshold() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("test", "testtype", "0", 0));
rated.add(new RatedDocument("test", "testtype", "1", 1));
@ -80,7 +79,7 @@ public class PrecisionAtNTests extends ESTestCase {
assertEquals(5, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException {
public void testPrecisionAtFiveCorrectIndex() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("test_other", "testtype", "0", Rating.RELEVANT.ordinal()));
rated.add(new RatedDocument("test_other", "testtype", "1", Rating.RELEVANT.ordinal()));
@ -94,7 +93,7 @@ public class PrecisionAtNTests extends ESTestCase {
assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException {
public void testPrecisionAtFiveCorrectType() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("test", "other_type", "0", Rating.RELEVANT.ordinal()));
rated.add(new RatedDocument("test", "other_type", "1", Rating.RELEVANT.ordinal()));
@ -107,6 +106,14 @@ public class PrecisionAtNTests extends ESTestCase {
assertEquals(3, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testNoRatedDocs() throws Exception {
List<RatedDocument> rated = new ArrayList<>();
EvalQueryQuality evaluated = (new PrecisionAtN(5)).evaluate("id", toSearchHits(rated, "test", "testtype"), rated);
assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(0, ((PrecisionAtN.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testParseFromXContent() throws IOException {
String xContent = " {\n"
+ " \"size\": 10,\n"