honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove maxAcceptableRank parameter from ReciprocalRank

This commit is contained in:
Christoph Büscher 2016-11-08 12:17:36 +01:00
parent c8d9d063ca
commit adb24333ca
2 changed files with 19 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java
View File

@ -45,8 +45,6 @@ import static org.elasticsearch.index.rankeval.RankedListQualityMetric.joinHitsW
public class ReciprocalRank implements RankedListQualityMetric { public class ReciprocalRank implements RankedListQualityMetric {
public static final String NAME = "reciprocal_rank"; public static final String NAME = "reciprocal_rank";
public static final int DEFAULT_MAX_ACCEPTABLE_RANK = Integer.MAX_VALUE;
private int maxAcceptableRank = DEFAULT_MAX_ACCEPTABLE_RANK;
/** ratings equal or above this value will be considered relevant. */ /** ratings equal or above this value will be considered relevant. */
private int relevantRatingThreshhold = 1; private int relevantRatingThreshhold = 1;
@ -58,19 +56,8 @@ public class ReciprocalRank implements RankedListQualityMetric {
// use defaults // use defaults
} }
/**
* @param maxAcceptableRank
* maximal acceptable rank. Must be positive.
*/
public ReciprocalRank(int maxAcceptableRank) {
if (maxAcceptableRank <= 0) {
throw new IllegalArgumentException("maximal acceptable rank needs to be positive but was [" + maxAcceptableRank + "]");
}
this.maxAcceptableRank = maxAcceptableRank;
}
public ReciprocalRank(StreamInput in) throws IOException { public ReciprocalRank(StreamInput in) throws IOException {
this.maxAcceptableRank = in.readInt(); this.relevantRatingThreshhold = in.readInt();
} }
@Override @Override
@ -78,21 +65,6 @@ public class ReciprocalRank implements RankedListQualityMetric {
return NAME; return NAME;
} }
/**
* @param maxAcceptableRank
* maximal acceptable rank. Must be positive.
*/
public void setMaxAcceptableRank(int maxAcceptableRank) {
if (maxAcceptableRank <= 0) {
throw new IllegalArgumentException("maximal acceptable rank needs to be positive but was [" + maxAcceptableRank + "]");
}
this.maxAcceptableRank = maxAcceptableRank;
}
public int getMaxAcceptableRank() {
return this.maxAcceptableRank;
}
/** /**
* Sets the rating threshold above which ratings are considered to be "relevant" for this metric. * Sets the rating threshold above which ratings are considered to be "relevant" for this metric.
* */ * */
@ -117,7 +89,7 @@ public class ReciprocalRank implements RankedListQualityMetric {
List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs); List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs);
int firstRelevant = -1; int firstRelevant = -1;
int rank = 1; int rank = 1;
for (RatedSearchHit hit : ratedHits.subList(0, Math.min(maxAcceptableRank, ratedHits.size()))) { for (RatedSearchHit hit : ratedHits) {
Optional<Integer> rating = hit.getRating(); Optional<Integer> rating = hit.getRating();
if (rating.isPresent()) { if (rating.isPresent()) {
if (rating.get() >= this.relevantRatingThreshhold) { if (rating.get() >= this.relevantRatingThreshhold) {
@ -137,16 +109,14 @@ public class ReciprocalRank implements RankedListQualityMetric {
@Override @Override
public void writeTo(StreamOutput out) throws IOException { public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(maxAcceptableRank); out.writeVInt(relevantRatingThreshhold);
} }
private static final ParseField MAX_RANK_FIELD = new ParseField("max_acceptable_rank");
private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold");
private static final ObjectParser<ReciprocalRank, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>( private static final ObjectParser<ReciprocalRank, ParseFieldMatcherSupplier> PARSER = new ObjectParser<>(
"reciprocal_rank", () -> new ReciprocalRank()); "reciprocal_rank", () -> new ReciprocalRank());
static { static {
PARSER.declareInt(ReciprocalRank::setMaxAcceptableRank, MAX_RANK_FIELD);
PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD); PARSER.declareInt(ReciprocalRank::setRelevantRatingThreshhold, RELEVANT_RATING_FIELD);
} }
@ -158,7 +128,7 @@ public class ReciprocalRank implements RankedListQualityMetric {
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(); builder.startObject();
builder.startObject(NAME); builder.startObject(NAME);
builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold);
builder.endObject(); builder.endObject();
builder.endObject(); builder.endObject();
return builder; return builder;
@ -173,12 +143,12 @@ public class ReciprocalRank implements RankedListQualityMetric {
return false; return false;
} }
ReciprocalRank other = (ReciprocalRank) obj; ReciprocalRank other = (ReciprocalRank) obj;
return Objects.equals(maxAcceptableRank, other.maxAcceptableRank); return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold);
} }
@Override @Override
public final int hashCode() { public final int hashCode() {
return Objects.hash(maxAcceptableRank); return Objects.hash(relevantRatingThreshhold);
} }
public static class Breakdown implements MetricDetails { public static class Breakdown implements MetricDetails {

35
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java
View File

@ -31,6 +31,7 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
import java.util.Vector; import java.util.Vector;
@ -39,16 +40,13 @@ public class ReciprocalRankTests extends ESTestCase {
public void testMaxAcceptableRank() { public void testMaxAcceptableRank() {
ReciprocalRank reciprocalRank = new ReciprocalRank(); ReciprocalRank reciprocalRank = new ReciprocalRank();
assertEquals(ReciprocalRank.DEFAULT_MAX_ACCEPTABLE_RANK, reciprocalRank.getMaxAcceptableRank());
int maxRank = randomIntBetween(1, 100); int searchHits = randomIntBetween(1, 50);
reciprocalRank.setMaxAcceptableRank(maxRank);
assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank());
SearchHit[] hits = createSearchHits(0, 9, "test", "type"); SearchHit[] hits = createSearchHits(0, searchHits, "test", "type");
List<RatedDocument> ratedDocs = new ArrayList<>(); List<RatedDocument> ratedDocs = new ArrayList<>();
int relevantAt = 5; int relevantAt = randomIntBetween(0, searchHits);
for (int i = 0; i < 10; i++) { for (int i = 0; i <= searchHits; i++) {
if (i == relevantAt) { if (i == relevantAt) {
ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.RELEVANT.ordinal())); ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i), Rating.RELEVANT.ordinal()));
} else { } else {
@ -58,19 +56,13 @@ public class ReciprocalRankTests extends ESTestCase {
int rankAtFirstRelevant = relevantAt + 1; int rankAtFirstRelevant = relevantAt + 1;
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
if (rankAtFirstRelevant <= maxRank) { assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE);
assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE); assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
// check that if we lower maxRank by one, we don't find any result and get 0.0 quality level // check that if we have fewer search hits than relevant doc position, we don't find any result and get 0.0 quality level
reciprocalRank = new ReciprocalRank(rankAtFirstRelevant - 1); reciprocalRank = new ReciprocalRank();
evaluation = reciprocalRank.evaluate("id", hits, ratedDocs); evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs);
assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE); assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
} else {
assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
assertEquals(-1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
}
} }
public void testEvaluationOneRelevantInResults() { public void testEvaluationOneRelevantInResults() {
@ -131,9 +123,8 @@ public class ReciprocalRankTests extends ESTestCase {
} }
public void testXContentRoundtrip() throws IOException { public void testXContentRoundtrip() throws IOException {
int position = randomIntBetween(0, 1000); ReciprocalRank testItem = new ReciprocalRank();
testItem.setRelevantRatingThreshhold(randomIntBetween(0, 20));
ReciprocalRank testItem = new ReciprocalRank(position);
XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem); XContentParser itemParser = RankEvalTestHelper.roundtrip(testItem);
itemParser.nextToken(); itemParser.nextToken();
itemParser.nextToken(); itemParser.nextToken();