Remove usage of types in rank_eval endpoint
This commit is contained in:
Christoph Büscher
Christoph Büscher
parent
c83ec1f133
commit
0a6c6ac360
|
|
@ -32,40 +32,30 @@ import java.util.Objects;
|
|||
public class DocumentKey implements Writeable, ToXContentObject {
|
||||
|
||||
private String docId;
|
||||
private String type;
|
||||
private String index;
|
||||
|
||||
void setIndex(String index) {
|
||||
this.index = index;
|
||||
}
|
||||
|
||||
void setType(String type) {
|
||||
this.type = type;
|
||||
}
|
||||
|
||||
void setDocId(String docId) {
|
||||
this.docId = docId;
|
||||
}
|
||||
|
||||
public DocumentKey(String index, String type, String docId) {
|
||||
public DocumentKey(String index, String docId) {
|
||||
if (Strings.isNullOrEmpty(index)) {
|
||||
throw new IllegalArgumentException("Index must be set for each rated document");
|
||||
}
|
||||
if(Strings.isNullOrEmpty(type)) {
|
||||
throw new IllegalArgumentException("Type must be set for each rated document");
|
||||
}
|
||||
if (Strings.isNullOrEmpty(docId)) {
|
||||
throw new IllegalArgumentException("DocId must be set for each rated document");
|
||||
}
|
||||
|
||||
this.index = index;
|
||||
this.type = type;
|
||||
this.docId = docId;
|
||||
}
|
||||
|
||||
public DocumentKey(StreamInput in) throws IOException {
|
||||
this.index = in.readString();
|
||||
this.type = in.readString();
|
||||
this.docId = in.readString();
|
||||
}
|
||||
|
||||
|
|
@ -73,10 +63,6 @@ public class DocumentKey implements Writeable, ToXContentObject {
|
|||
return index;
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return type;
|
||||
}
|
||||
|
||||
public String getDocID() {
|
||||
return docId;
|
||||
}
|
||||
|
|
@ -84,7 +70,6 @@ public class DocumentKey implements Writeable, ToXContentObject {
|
|||
@Override
|
||||
public void writeTo(StreamOutput out) throws IOException {
|
||||
out.writeString(index);
|
||||
out.writeString(type);
|
||||
out.writeString(docId);
|
||||
}
|
||||
|
||||
|
|
@ -97,21 +82,18 @@ public class DocumentKey implements Writeable, ToXContentObject {
|
|||
return false;
|
||||
}
|
||||
DocumentKey other = (DocumentKey) obj;
|
||||
return Objects.equals(index, other.index) &&
|
||||
Objects.equals(type, other.type) &&
|
||||
Objects.equals(docId, other.docId);
|
||||
return Objects.equals(index, other.index) && Objects.equals(docId, other.docId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public final int hashCode() {
|
||||
return Objects.hash(index, type, docId);
|
||||
return Objects.hash(index, docId);
|
||||
}
|
||||
|
||||
@Override
|
||||
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
builder.startObject();
|
||||
builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), index);
|
||||
builder.field(RatedDocument.TYPE_FIELD.getPreferredName(), type);
|
||||
builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), docId);
|
||||
builder.endObject();
|
||||
return builder;
|
||||
|
|
|
|||
|
|
@ -35,29 +35,23 @@ import java.util.Optional;
|
|||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* Classes implementing this interface provide a means to compute the quality of
|
||||
* a result list returned by some search.
|
||||
*
|
||||
* RelevancyLevel specifies the type of object determining the relevancy level
|
||||
* of some known docid.
|
||||
* Classes implementing this interface provide a means to compute the quality of a result list returned by some search.
|
||||
*/
|
||||
public interface RankedListQualityMetric extends ToXContent, NamedWriteable {
|
||||
|
||||
/**
|
||||
* Returns a single metric representing the ranking quality of a set of
|
||||
* returned documents wrt. to a set of document Ids labeled as relevant for
|
||||
* this search.
|
||||
* Returns a single metric representing the ranking quality of a set of returned
|
||||
* documents wrt. to a set of document Ids labeled as relevant for this search.
|
||||
*
|
||||
* @param taskId
|
||||
* the id of the query for which the ranking is currently
|
||||
* evaluated
|
||||
* the id of the query for which the ranking is currently evaluated
|
||||
* @param hits
|
||||
* the result hits as returned by a search request
|
||||
* @param ratedDocs
|
||||
* the documents that were ranked by human annotators for this
|
||||
* query case
|
||||
* @return some metric representing the quality of the result hit list wrt.
|
||||
* to relevant doc ids.
|
||||
* the documents that were ranked by human annotators for this query
|
||||
* case
|
||||
* @return some metric representing the quality of the result hit list wrt. to
|
||||
* relevant doc ids.
|
||||
*/
|
||||
EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List<RatedDocument> ratedDocs);
|
||||
|
||||
|
|
@ -65,8 +59,7 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable {
|
|||
RankedListQualityMetric rc;
|
||||
Token token = parser.nextToken();
|
||||
if (token != XContentParser.Token.FIELD_NAME) {
|
||||
throw new ParsingException(parser.getTokenLocation(),
|
||||
"[_na] missing required metric name");
|
||||
throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name");
|
||||
}
|
||||
String metricName = parser.currentName();
|
||||
|
||||
|
|
@ -82,8 +75,7 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable {
|
|||
rc = DiscountedCumulativeGain.fromXContent(parser);
|
||||
break;
|
||||
default:
|
||||
throw new ParsingException(parser.getTokenLocation(),
|
||||
"[_na] unknown query metric name [{}]", metricName);
|
||||
throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName);
|
||||
}
|
||||
if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
|
||||
// if we are at END_OBJECT, move to the next one...
|
||||
|
|
@ -92,14 +84,13 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable {
|
|||
return rc;
|
||||
}
|
||||
|
||||
static List<RatedSearchHit> joinHitsWithRatings(SearchHit[] hits,
|
||||
List<RatedDocument> ratedDocs) {
|
||||
static List<RatedSearchHit> joinHitsWithRatings(SearchHit[] hits, List<RatedDocument> ratedDocs) {
|
||||
// join hits with rated documents
|
||||
Map<DocumentKey, RatedDocument> ratedDocumentMap = ratedDocs.stream()
|
||||
.collect(Collectors.toMap(RatedDocument::getKey, item -> item));
|
||||
List<RatedSearchHit> ratedSearchHits = new ArrayList<>(hits.length);
|
||||
for (SearchHit hit : hits) {
|
||||
DocumentKey key = new DocumentKey(hit.getIndex(), hit.getType(), hit.getId());
|
||||
DocumentKey key = new DocumentKey(hit.getIndex(), hit.getId());
|
||||
RatedDocument ratedDoc = ratedDocumentMap.get(key);
|
||||
if (ratedDoc != null) {
|
||||
ratedSearchHits.add(new RatedSearchHit(hit, Optional.of(ratedDoc.getRating())));
|
||||
|
|
@ -112,16 +103,12 @@ public interface RankedListQualityMetric extends ToXContent, NamedWriteable {
|
|||
|
||||
static List<DocumentKey> filterUnknownDocuments(List<RatedSearchHit> ratedHits) {
|
||||
// join hits with rated documents
|
||||
List<DocumentKey> unknownDocs = ratedHits.stream()
|
||||
.filter(hit -> hit.getRating().isPresent() == false)
|
||||
.map(hit -> new DocumentKey(hit.getSearchHit().getIndex(),
|
||||
hit.getSearchHit().getType(), hit.getSearchHit().getId()))
|
||||
.collect(Collectors.toList());
|
||||
List<DocumentKey> unknownDocs = ratedHits.stream().filter(hit -> hit.getRating().isPresent() == false)
|
||||
.map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), hit.getSearchHit().getId())).collect(Collectors.toList());
|
||||
return unknownDocs;
|
||||
}
|
||||
|
||||
default double combine(Collection<EvalQueryQuality> partialResults) {
|
||||
return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum()
|
||||
/ partialResults.size();
|
||||
return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -34,21 +34,18 @@ import java.util.Objects;
|
|||
|
||||
/**
|
||||
* A document ID and its rating for the query QA use case.
|
||||
* */
|
||||
*/
|
||||
public class RatedDocument implements Writeable, ToXContentObject {
|
||||
|
||||
public static final ParseField RATING_FIELD = new ParseField("rating");
|
||||
public static final ParseField DOC_ID_FIELD = new ParseField("_id");
|
||||
public static final ParseField TYPE_FIELD = new ParseField("_type");
|
||||
public static final ParseField INDEX_FIELD = new ParseField("_index");
|
||||
|
||||
private static final ConstructingObjectParser<RatedDocument, Void> PARSER =
|
||||
new ConstructingObjectParser<>("rated_document",
|
||||
a -> new RatedDocument((String) a[0], (String) a[1], (String) a[2], (Integer) a[3]));
|
||||
private static final ConstructingObjectParser<RatedDocument, Void> PARSER = new ConstructingObjectParser<>("rated_document",
|
||||
a -> new RatedDocument((String) a[0], (String) a[1], (Integer) a[2]));
|
||||
|
||||
static {
|
||||
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD);
|
||||
PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD);
|
||||
PARSER.declareString(ConstructingObjectParser.constructorArg(), DOC_ID_FIELD);
|
||||
PARSER.declareInt(ConstructingObjectParser.constructorArg(), RATING_FIELD);
|
||||
}
|
||||
|
|
@ -56,8 +53,8 @@ public class RatedDocument implements Writeable, ToXContentObject {
|
|||
private int rating;
|
||||
private DocumentKey key;
|
||||
|
||||
public RatedDocument(String index, String type, String docId, int rating) {
|
||||
this(new DocumentKey(index, type, docId), rating);
|
||||
public RatedDocument(String index, String docId, int rating) {
|
||||
this(new DocumentKey(index, docId), rating);
|
||||
}
|
||||
|
||||
public RatedDocument(StreamInput in) throws IOException {
|
||||
|
|
@ -78,10 +75,6 @@ public class RatedDocument implements Writeable, ToXContentObject {
|
|||
return key.getIndex();
|
||||
}
|
||||
|
||||
public String getType() {
|
||||
return key.getType();
|
||||
}
|
||||
|
||||
public String getDocID() {
|
||||
return key.getDocID();
|
||||
}
|
||||
|
|
@ -104,7 +97,6 @@ public class RatedDocument implements Writeable, ToXContentObject {
|
|||
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
|
||||
builder.startObject();
|
||||
builder.field(INDEX_FIELD.getPreferredName(), key.getIndex());
|
||||
builder.field(TYPE_FIELD.getPreferredName(), key.getType());
|
||||
builder.field(DOC_ID_FIELD.getPreferredName(), key.getDocID());
|
||||
builder.field(RATING_FIELD.getPreferredName(), rating);
|
||||
builder.endObject();
|
||||
|
|
@ -125,8 +117,7 @@ public class RatedDocument implements Writeable, ToXContentObject {
|
|||
return false;
|
||||
}
|
||||
RatedDocument other = (RatedDocument) obj;
|
||||
return Objects.equals(key, other.key) &&
|
||||
Objects.equals(rating, other.rating);
|
||||
return Objects.equals(key, other.key) && Objects.equals(rating, other.rating);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
|||
|
|
@ -54,7 +54,6 @@ import java.util.Set;
|
|||
public class RatedRequest implements Writeable, ToXContentObject {
|
||||
private String id;
|
||||
private List<String> indices = new ArrayList<>();
|
||||
private List<String> types = new ArrayList<>();
|
||||
private List<String> summaryFields = new ArrayList<>();
|
||||
/** Collection of rated queries for this query QA specification. */
|
||||
private List<RatedDocument> ratedDocs = new ArrayList<>();
|
||||
|
|
@ -93,7 +92,7 @@ public class RatedRequest implements Writeable, ToXContentObject {
|
|||
"If template parameters are supplied need to set id of template to apply "
|
||||
+ "them to too.");
|
||||
}
|
||||
// No documents with same _index/_type/id allowed.
|
||||
// No documents with same _index/id allowed.
|
||||
Set<DocumentKey> docKeys = new HashSet<>();
|
||||
for (RatedDocument doc : ratedDocs) {
|
||||
if (docKeys.add(doc.getKey()) == false) {
|
||||
|
|
@ -131,11 +130,6 @@ public class RatedRequest implements Writeable, ToXContentObject {
|
|||
for (int i = 0; i < indicesSize; i++) {
|
||||
this.indices.add(in.readString());
|
||||
}
|
||||
int typesSize = in.readInt();
|
||||
types = new ArrayList<>(typesSize);
|
||||
for (int i = 0; i < typesSize; i++) {
|
||||
this.types.add(in.readString());
|
||||
}
|
||||
int intentSize = in.readInt();
|
||||
ratedDocs = new ArrayList<>(intentSize);
|
||||
for (int i = 0; i < intentSize; i++) {
|
||||
|
|
@ -159,10 +153,6 @@ public class RatedRequest implements Writeable, ToXContentObject {
|
|||
for (String index : indices) {
|
||||
out.writeString(index);
|
||||
}
|
||||
out.writeInt(types.size());
|
||||
for (String type : types) {
|
||||
out.writeString(type);
|
||||
}
|
||||
out.writeInt(ratedDocs.size());
|
||||
for (RatedDocument ratedDoc : ratedDocs) {
|
||||
ratedDoc.writeTo(out);
|
||||
|
|
@ -187,14 +177,6 @@ public class RatedRequest implements Writeable, ToXContentObject {
|
|||
return indices;
|
||||
}
|
||||
|
||||
public void setTypes(List<String> types) {
|
||||
this.types = types;
|
||||
}
|
||||
|
||||
public List<String> getTypes() {
|
||||
return types;
|
||||
}
|
||||
|
||||
/** Returns a user supplied spec id for easier referencing. */
|
||||
public String getId() {
|
||||
return id;
|
||||
|
|
@ -314,7 +296,7 @@ public class RatedRequest implements Writeable, ToXContentObject {
|
|||
RatedRequest other = (RatedRequest) obj;
|
||||
|
||||
return Objects.equals(id, other.id) && Objects.equals(testRequest, other.testRequest)
|
||||
&& Objects.equals(indices, other.indices) && Objects.equals(types, other.types)
|
||||
&& Objects.equals(indices, other.indices)
|
||||
&& Objects.equals(summaryFields, other.summaryFields)
|
||||
&& Objects.equals(ratedDocs, other.ratedDocs)
|
||||
&& Objects.equals(params, other.params)
|
||||
|
|
@ -323,7 +305,7 @@ public class RatedRequest implements Writeable, ToXContentObject {
|
|||
|
||||
@Override
|
||||
public final int hashCode() {
|
||||
return Objects.hash(id, testRequest, indices, types, summaryFields, ratedDocs, params,
|
||||
return Objects.hash(id, testRequest, indices, summaryFields, ratedDocs, params,
|
||||
templateId);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -135,13 +135,13 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
|
|||
"quality_level": 0.4,
|
||||
"unknown_docs": {
|
||||
"amsterdam_query": [
|
||||
{ "index" : "test", "type" : "my_type", "doc_id" : "21"},
|
||||
{ "index" : "test", "type" : "my_type", "doc_id" : "5"},
|
||||
{ "index" : "test", "type" : "my_type", "doc_id" : "9"}
|
||||
{ "index" : "test", "doc_id" : "21"},
|
||||
{ "index" : "test", "doc_id" : "5"},
|
||||
{ "index" : "test", "doc_id" : "9"}
|
||||
]
|
||||
}, {
|
||||
"berlin_query": [
|
||||
{ "index" : "test", "type" : "my_type", "doc_id" : "42"}
|
||||
{ "index" : "test", "doc_id" : "42"}
|
||||
]
|
||||
}
|
||||
}]
|
||||
|
|
@ -157,8 +157,6 @@ public class RestRankEvalAction extends BaseRestHandler {
|
|||
controller.registerHandler(POST, "/_rank_eval", this);
|
||||
controller.registerHandler(GET, "/{index}/_rank_eval", this);
|
||||
controller.registerHandler(POST, "/{index}/_rank_eval", this);
|
||||
controller.registerHandler(GET, "/{index}/{type}/_rank_eval", this);
|
||||
controller.registerHandler(POST, "/{index}/{type}/_rank_eval", this);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -176,13 +174,10 @@ public class RestRankEvalAction extends BaseRestHandler {
|
|||
XContentParser parser) {
|
||||
List<String> indices = Arrays
|
||||
.asList(Strings.splitStringByCommaToArray(request.param("index")));
|
||||
List<String> types = Arrays
|
||||
.asList(Strings.splitStringByCommaToArray(request.param("type")));
|
||||
RankEvalSpec spec = null;
|
||||
spec = RankEvalSpec.parse(parser);
|
||||
for (RatedRequest specification : spec.getRatedRequests()) {
|
||||
specification.setIndices(indices);
|
||||
specification.setTypes(types);
|
||||
}
|
||||
;
|
||||
|
||||
|
|
|
|||
|
|
@ -128,10 +128,6 @@ public class TransportRankEvalAction
|
|||
indices = ratedRequest.getIndices().toArray(indices);
|
||||
SearchRequest templatedRequest = new SearchRequest(indices, ratedSearchSource);
|
||||
|
||||
String[] types = new String[ratedRequest.getTypes().size()];
|
||||
types = ratedRequest.getTypes().toArray(types);
|
||||
templatedRequest.types(types);
|
||||
|
||||
RequestTask task = new RequestTask(templatedRequest, searchListener);
|
||||
taskQueue.add(task);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,13 +43,12 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
/**
|
||||
* Assuming the docs are ranked in the following order:
|
||||
*
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1)
|
||||
* / log_2(rank + 1)
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) /
|
||||
* log_2(rank + 1)
|
||||
* -------------------------------------------------------------------------------------------
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563
|
||||
* | 1.8927892607143721 3 | 3 | 7.0 | 2.0 | 3.5 4 | 0 | 0.0
|
||||
* | 2.321928094887362 | 0.0 5 | 1 | 1.0 | 2.584962500721156
|
||||
* | 0.38685280723454163 6 | 2 | 3.0 | 2.807354922057604
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
|
||||
* 3 | 3 | 7.0 | 2.0 | 3.5 4 | 0 | 0.0 | 2.321928094887362 | 0.0 5 | 1 | 1.0
|
||||
* | 2.584962500721156 | 0.38685280723454163 6 | 2 | 3.0 | 2.807354922057604
|
||||
* | 1.0686215613240666
|
||||
*
|
||||
* dcg = 13.84826362927298 (sum of last column)
|
||||
|
|
@ -59,45 +58,41 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 };
|
||||
SearchHit[] hits = new SearchHit[6];
|
||||
for (int i = 0; i < 6; i++) {
|
||||
rated.add(new RatedDocument("index", "type", Integer.toString(i), relevanceRatings[i]));
|
||||
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"),
|
||||
Collections.emptyMap());
|
||||
rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
|
||||
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
|
||||
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
|
||||
}
|
||||
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
|
||||
assertEquals(13.84826362927298, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
|
||||
|
||||
/**
|
||||
* Check with normalization: to get the maximal possible dcg, sort
|
||||
* documents by relevance in descending order
|
||||
* Check with normalization: to get the maximal possible dcg, sort documents by
|
||||
* relevance in descending order
|
||||
*
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank)
|
||||
* - 1) / log_2(rank + 1)
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) /
|
||||
* log_2(rank + 1)
|
||||
* ---------------------------------------------------------------------------------------
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 3 | 7.0 |
|
||||
* 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0 | 1.5 4 | 2
|
||||
* | 3.0 | 2.321928094887362 | 1.2920296742201793 5 | 1 | 1.0 |
|
||||
* 2.584962500721156 | 0.38685280723454163 6 | 0 | 0.0 |
|
||||
* 2.807354922057604 | 0.0
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
|
||||
* 3 | 2 | 3.0 | 2.0 | 1.5 4 | 2 | 3.0 | 2.321928094887362
|
||||
* | 1.2920296742201793 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 6
|
||||
* | 0 | 0.0 | 2.807354922057604 | 0.0
|
||||
*
|
||||
* idcg = 14.595390756454922 (sum of last column)
|
||||
*/
|
||||
dcg.setNormalize(true);
|
||||
assertEquals(13.84826362927298 / 14.595390756454922,
|
||||
dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
|
||||
assertEquals(13.84826362927298 / 14.595390756454922, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
|
||||
}
|
||||
|
||||
/**
|
||||
* This tests metric when some documents in the search result don't have a
|
||||
* rating provided by the user.
|
||||
*
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1)
|
||||
* / log_2(rank + 1)
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) /
|
||||
* log_2(rank + 1)
|
||||
* -------------------------------------------------------------------------------------------
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563
|
||||
* | 1.8927892607143721 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a 5
|
||||
* | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a |
|
||||
* n/a
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
|
||||
* 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a 5 | 1 | 1.0
|
||||
* | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a
|
||||
*
|
||||
* dcg = 12.779642067948913 (sum of last column)
|
||||
*/
|
||||
|
|
@ -108,12 +103,10 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
for (int i = 0; i < 6; i++) {
|
||||
if (i < relevanceRatings.length) {
|
||||
if (relevanceRatings[i] != null) {
|
||||
rated.add(new RatedDocument("index", "type", Integer.toString(i),
|
||||
relevanceRatings[i]));
|
||||
rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
|
||||
}
|
||||
}
|
||||
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"),
|
||||
Collections.emptyMap());
|
||||
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
|
||||
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
|
||||
}
|
||||
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
|
||||
|
|
@ -122,36 +115,34 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size());
|
||||
|
||||
/**
|
||||
* Check with normalization: to get the maximal possible dcg, sort
|
||||
* documents by relevance in descending order
|
||||
* Check with normalization: to get the maximal possible dcg, sort documents by
|
||||
* relevance in descending order
|
||||
*
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank)
|
||||
* - 1) / log_2(rank + 1)
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) /
|
||||
* log_2(rank + 1)
|
||||
* ----------------------------------------------------------------------------------------
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 3 | 7.0 |
|
||||
* 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0 | 1.5 4 | 1
|
||||
* | 1.0 | 2.321928094887362 | 0.43067655807339 5 | n.a | n.a | n.a.
|
||||
* | n.a. 6 | n.a | n.a | n.a | n.a
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
|
||||
* 3 | 2 | 3.0 | 2.0 | 1.5 4 | 1 | 1.0 | 2.321928094887362 | 0.43067655807339
|
||||
* 5 | n.a | n.a | n.a. | n.a. 6 | n.a | n.a | n.a | n.a
|
||||
*
|
||||
* idcg = 13.347184833073591 (sum of last column)
|
||||
*/
|
||||
dcg.setNormalize(true);
|
||||
assertEquals(12.779642067948913 / 13.347184833073591,
|
||||
dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
|
||||
assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), 0.00001);
|
||||
}
|
||||
|
||||
/**
|
||||
* This tests that normalization works as expected when there are more rated
|
||||
* documents than search hits because we restrict DCG to be calculated at
|
||||
* the fourth position
|
||||
* documents than search hits because we restrict DCG to be calculated at the
|
||||
* fourth position
|
||||
*
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1)
|
||||
* / log_2(rank + 1)
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) /
|
||||
* log_2(rank + 1)
|
||||
* -------------------------------------------------------------------------------------------
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563
|
||||
* | 1.8927892607143721 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a
|
||||
* ----------------------------------------------------------------- 5 | 1
|
||||
* | 1.0 | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
|
||||
* 3 | 3 | 7.0 | 2.0 | 3.5 4 | n/a | n/a | n/a | n/a
|
||||
* ----------------------------------------------------------------- 5 | 1 | 1.0
|
||||
* | 2.584962500721156 | 0.38685280723454163 6 | n/a | n/a | n/a | n/a
|
||||
*
|
||||
* dcg = 12.392789260714371 (sum of last column until position 4)
|
||||
*/
|
||||
|
|
@ -161,16 +152,14 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
for (int i = 0; i < 6; i++) {
|
||||
if (i < relevanceRatings.length) {
|
||||
if (relevanceRatings[i] != null) {
|
||||
ratedDocs.add(new RatedDocument("index", "type", Integer.toString(i),
|
||||
relevanceRatings[i]));
|
||||
ratedDocs.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
|
||||
}
|
||||
}
|
||||
}
|
||||
// only create four hits
|
||||
SearchHit[] hits = new SearchHit[4];
|
||||
for (int i = 0; i < 4; i++) {
|
||||
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"),
|
||||
Collections.emptyMap());
|
||||
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
|
||||
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
|
||||
}
|
||||
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
|
||||
|
|
@ -179,28 +168,25 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size());
|
||||
|
||||
/**
|
||||
* Check with normalization: to get the maximal possible dcg, sort
|
||||
* documents by relevance in descending order
|
||||
* Check with normalization: to get the maximal possible dcg, sort documents by
|
||||
* relevance in descending order
|
||||
*
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank)
|
||||
* - 1) / log_2(rank + 1)
|
||||
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) /
|
||||
* log_2(rank + 1)
|
||||
* ---------------------------------------------------------------------------------------
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 3 | 7.0 |
|
||||
* 1.5849625007211563 | 4.416508275000202 3 | 2 | 3.0 | 2.0 | 1.5 4 | 1
|
||||
* | 1.0 | 2.321928094887362 | 0.43067655807339
|
||||
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
|
||||
* 3 | 2 | 3.0 | 2.0 | 1.5 4 | 1 | 1.0 | 2.321928094887362 | 0.43067655807339
|
||||
* ---------------------------------------------------------------------------------------
|
||||
* 5 | n.a | n.a | n.a. | n.a. 6 | n.a | n.a | n.a | n.a
|
||||
*
|
||||
* idcg = 13.347184833073591 (sum of last column)
|
||||
*/
|
||||
dcg.setNormalize(true);
|
||||
assertEquals(12.392789260714371 / 13.347184833073591,
|
||||
dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001);
|
||||
assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), 0.00001);
|
||||
}
|
||||
|
||||
public void testParseFromXContent() throws IOException {
|
||||
String xContent = " {\n" + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n"
|
||||
+ "}";
|
||||
String xContent = " {\n" + " \"unknown_doc_rating\": 2,\n" + " \"normalize\": true\n" + "}";
|
||||
try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
|
||||
DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser);
|
||||
assertEquals(2, dcgAt.getUnknownDocRating().intValue());
|
||||
|
|
@ -218,8 +204,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
public void testXContentRoundtrip() throws IOException {
|
||||
DiscountedCumulativeGain testItem = createTestItem();
|
||||
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
|
||||
XContentBuilder shuffled = shuffleXContent(
|
||||
testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
try (XContentParser itemParser = createParser(shuffled)) {
|
||||
itemParser.nextToken();
|
||||
itemParser.nextToken();
|
||||
|
|
@ -232,8 +217,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
|
||||
public void testSerialization() throws IOException {
|
||||
DiscountedCumulativeGain original = createTestItem();
|
||||
DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original,
|
||||
DiscountedCumulativeGain::new);
|
||||
DiscountedCumulativeGain deserialized = RankEvalTestHelper.copy(original, DiscountedCumulativeGain::new);
|
||||
assertEquals(deserialized, original);
|
||||
assertEquals(deserialized.hashCode(), original.hashCode());
|
||||
assertNotSame(deserialized, original);
|
||||
|
|
@ -254,8 +238,7 @@ public class DiscountedCumulativeGainTests extends ESTestCase {
|
|||
|
||||
List<Runnable> mutators = new ArrayList<>();
|
||||
mutators.add(() -> gain.setNormalize(!original.getNormalize()));
|
||||
mutators.add(() -> gain.setUnknownDocRating(
|
||||
randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10))));
|
||||
mutators.add(() -> gain.setUnknownDocRating(randomValueOtherThan(unknownDocRating, () -> randomIntBetween(0, 10))));
|
||||
randomFrom(mutators).run();
|
||||
return gain;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,14 +23,12 @@ import org.elasticsearch.test.ESTestCase;
|
|||
|
||||
import java.io.IOException;
|
||||
|
||||
|
||||
public class DocumentKeyTests extends ESTestCase {
|
||||
|
||||
static DocumentKey createRandomRatedDocumentKey() {
|
||||
String index = randomAlphaOfLengthBetween(1, 10);
|
||||
String type = randomAlphaOfLengthBetween(1, 10);
|
||||
String docId = randomAlphaOfLengthBetween(1, 10);
|
||||
return new DocumentKey(index, type, docId);
|
||||
return new DocumentKey(index, docId);
|
||||
}
|
||||
|
||||
public DocumentKey createTestItem() {
|
||||
|
|
@ -39,28 +37,24 @@ public class DocumentKeyTests extends ESTestCase {
|
|||
|
||||
public DocumentKey mutateTestItem(DocumentKey original) {
|
||||
String index = original.getIndex();
|
||||
String type = original.getType();
|
||||
String docId = original.getDocID();
|
||||
switch (randomIntBetween(0, 2)) {
|
||||
switch (randomIntBetween(0, 1)) {
|
||||
case 0:
|
||||
index = index + "_";
|
||||
break;
|
||||
case 1:
|
||||
type = type + "_";
|
||||
break;
|
||||
case 2:
|
||||
docId = docId + "_";
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException("The test should only allow three parameters mutated");
|
||||
throw new IllegalStateException("The test should only allow two parameters mutated");
|
||||
}
|
||||
return new DocumentKey(index, type, docId);
|
||||
return new DocumentKey(index, docId);
|
||||
}
|
||||
|
||||
public void testEqualsAndHash() throws IOException {
|
||||
DocumentKey testItem = createRandomRatedDocumentKey();
|
||||
RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem),
|
||||
new DocumentKey(testItem.getIndex(), testItem.getType(), testItem.getDocID()));
|
||||
new DocumentKey(testItem.getIndex(), testItem.getDocID()));
|
||||
}
|
||||
|
||||
public void testSerialization() throws IOException {
|
||||
|
|
@ -70,6 +64,4 @@ public class DocumentKeyTests extends ESTestCase {
|
|||
assertEquals(deserialized.hashCode(), original.hashCode());
|
||||
assertNotSame(deserialized, original);
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -42,96 +42,72 @@ public class PrecisionTests extends ESTestCase {
|
|||
|
||||
public void testPrecisionAtFiveCalculation() {
|
||||
List<RatedDocument> rated = new ArrayList<>();
|
||||
rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id",
|
||||
toSearchHits(rated, "test", "testtype"), rated);
|
||||
rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal()));
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test"), rated);
|
||||
assertEquals(1, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(1,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(1, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
}
|
||||
|
||||
public void testPrecisionAtFiveIgnoreOneResult() {
|
||||
List<RatedDocument> rated = new ArrayList<>();
|
||||
rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal()));
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id",
|
||||
toSearchHits(rated, "test", "testtype"), rated);
|
||||
rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "2", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "3", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "4", Rating.IRRELEVANT.ordinal()));
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated, "test"), rated);
|
||||
assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(4,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(4, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
}
|
||||
|
||||
/**
|
||||
* test that the relevant rating threshold can be set to something larger
|
||||
* than 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant,
|
||||
* doc 3 and 4 to be relevant
|
||||
* test that the relevant rating threshold can be set to something larger than
|
||||
* 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and
|
||||
* 4 to be relevant
|
||||
*/
|
||||
public void testPrecisionAtFiveRelevanceThreshold() {
|
||||
List<RatedDocument> rated = new ArrayList<>();
|
||||
rated.add(new RatedDocument("test", "testtype", "0", 0));
|
||||
rated.add(new RatedDocument("test", "testtype", "1", 1));
|
||||
rated.add(new RatedDocument("test", "testtype", "2", 2));
|
||||
rated.add(new RatedDocument("test", "testtype", "3", 3));
|
||||
rated.add(new RatedDocument("test", "testtype", "4", 4));
|
||||
rated.add(new RatedDocument("test", "0", 0));
|
||||
rated.add(new RatedDocument("test", "1", 1));
|
||||
rated.add(new RatedDocument("test", "2", 2));
|
||||
rated.add(new RatedDocument("test", "3", 3));
|
||||
rated.add(new RatedDocument("test", "4", 4));
|
||||
Precision precisionAtN = new Precision();
|
||||
precisionAtN.setRelevantRatingThreshhold(2);
|
||||
EvalQueryQuality evaluated = precisionAtN.evaluate("id",
|
||||
toSearchHits(rated, "test", "testtype"), rated);
|
||||
EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test"), rated);
|
||||
assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(3,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
}
|
||||
|
||||
public void testPrecisionAtFiveCorrectIndex() {
|
||||
List<RatedDocument> rated = new ArrayList<>();
|
||||
rated.add(new RatedDocument("test_other", "testtype", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test_other", "testtype", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test_other", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test_other", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "2", Rating.IRRELEVANT.ordinal()));
|
||||
// the following search hits contain only the last three documents
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id",
|
||||
toSearchHits(rated.subList(2, 5), "test", "testtype"), rated);
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id", toSearchHits(rated.subList(2, 5), "test"), rated);
|
||||
assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(2,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
}
|
||||
|
||||
public void testPrecisionAtFiveCorrectType() {
|
||||
List<RatedDocument> rated = new ArrayList<>();
|
||||
rated.add(new RatedDocument("test", "other_type", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "other_type", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "2", Rating.IRRELEVANT.ordinal()));
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id",
|
||||
toSearchHits(rated.subList(2, 5), "test", "testtype"), rated);
|
||||
assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(2,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
}
|
||||
|
||||
public void testIgnoreUnlabeled() {
|
||||
List<RatedDocument> rated = new ArrayList<>();
|
||||
rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "0", Rating.RELEVANT.ordinal()));
|
||||
rated.add(new RatedDocument("test", "1", Rating.RELEVANT.ordinal()));
|
||||
// add an unlabeled search hit
|
||||
SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test", "testtype"), 3);
|
||||
SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3);
|
||||
searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap());
|
||||
searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
|
||||
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id", searchHits, rated);
|
||||
assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(2,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(3, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
|
||||
// also try with setting `ignore_unlabeled`
|
||||
|
|
@ -139,8 +115,7 @@ public class PrecisionTests extends ESTestCase {
|
|||
prec.setIgnoreUnlabeled(true);
|
||||
evaluated = prec.evaluate("id", searchHits, rated);
|
||||
assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(2,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(2, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
}
|
||||
|
||||
|
|
@ -150,11 +125,9 @@ public class PrecisionTests extends ESTestCase {
|
|||
hits[i] = new SearchHit(i, i + "", new Text("type"), Collections.emptyMap());
|
||||
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
|
||||
}
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits,
|
||||
Collections.emptyList());
|
||||
EvalQueryQuality evaluated = (new Precision()).evaluate("id", hits, Collections.emptyList());
|
||||
assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(0,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(5, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
|
||||
// also try with setting `ignore_unlabeled`
|
||||
|
|
@ -162,8 +135,7 @@ public class PrecisionTests extends ESTestCase {
|
|||
prec.setIgnoreUnlabeled(true);
|
||||
evaluated = prec.evaluate("id", hits, Collections.emptyList());
|
||||
assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
|
||||
assertEquals(0,
|
||||
((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
|
||||
assertEquals(0, ((Precision.Breakdown) evaluated.getMetricDetails()).getRetrieved());
|
||||
}
|
||||
|
||||
|
|
@ -201,8 +173,7 @@ public class PrecisionTests extends ESTestCase {
|
|||
public void testXContentRoundtrip() throws IOException {
|
||||
Precision testItem = createTestItem();
|
||||
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
|
||||
XContentBuilder shuffled = shuffleXContent(
|
||||
testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
try (XContentParser itemParser = createParser(shuffled)) {
|
||||
itemParser.nextToken();
|
||||
itemParser.nextToken();
|
||||
|
|
@ -223,8 +194,7 @@ public class PrecisionTests extends ESTestCase {
|
|||
|
||||
public void testEqualsAndHash() throws IOException {
|
||||
Precision testItem = createTestItem();
|
||||
RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem),
|
||||
RankEvalTestHelper.copy(testItem, Precision::new));
|
||||
RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, Precision::new));
|
||||
}
|
||||
|
||||
private static Precision mutateTestItem(Precision original) {
|
||||
|
|
@ -236,16 +206,15 @@ public class PrecisionTests extends ESTestCase {
|
|||
|
||||
List<Runnable> mutators = new ArrayList<>();
|
||||
mutators.add(() -> precision.setIgnoreUnlabeled(!ignoreUnlabeled));
|
||||
mutators.add(() -> precision.setRelevantRatingThreshhold(
|
||||
randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))));
|
||||
mutators.add(() -> precision.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10))));
|
||||
randomFrom(mutators).run();
|
||||
return precision;
|
||||
}
|
||||
|
||||
private static SearchHit[] toSearchHits(List<RatedDocument> rated, String index, String type) {
|
||||
private static SearchHit[] toSearchHits(List<RatedDocument> rated, String index) {
|
||||
SearchHit[] hits = new SearchHit[rated.size()];
|
||||
for (int i = 0; i < rated.size(); i++) {
|
||||
hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap());
|
||||
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap());
|
||||
hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null));
|
||||
}
|
||||
return hits;
|
||||
|
|
|
|||
|
|
@ -65,7 +65,6 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
|
||||
public void testPrecisionAtRequest() {
|
||||
List<String> indices = Arrays.asList(new String[] { "test" });
|
||||
List<String> types = Arrays.asList(new String[] { "testtype" });
|
||||
|
||||
List<RatedRequest> specifications = new ArrayList<>();
|
||||
SearchSourceBuilder testQuery = new SearchSourceBuilder();
|
||||
|
|
@ -73,14 +72,12 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
|
||||
createRelevant("2", "3", "4", "5"), testQuery);
|
||||
amsterdamRequest.setIndices(indices);
|
||||
amsterdamRequest.setTypes(types);
|
||||
amsterdamRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" }));
|
||||
|
||||
specifications.add(amsterdamRequest);
|
||||
RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"),
|
||||
testQuery);
|
||||
berlinRequest.setIndices(indices);
|
||||
berlinRequest.setTypes(types);
|
||||
berlinRequest.setSummaryFields(Arrays.asList(new String[] { "text", "title" }));
|
||||
|
||||
specifications.add(berlinRequest);
|
||||
|
|
@ -135,7 +132,6 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
*/
|
||||
public void testBadQuery() {
|
||||
List<String> indices = Arrays.asList(new String[] { "test" });
|
||||
List<String> types = Arrays.asList(new String[] { "testtype" });
|
||||
|
||||
List<RatedRequest> specifications = new ArrayList<>();
|
||||
SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder();
|
||||
|
|
@ -143,7 +139,6 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
|
||||
createRelevant("2", "3", "4", "5"), amsterdamQuery);
|
||||
amsterdamRequest.setIndices(indices);
|
||||
amsterdamRequest.setTypes(types);
|
||||
specifications.add(amsterdamRequest);
|
||||
|
||||
SearchSourceBuilder brokenQuery = new SearchSourceBuilder();
|
||||
|
|
@ -151,7 +146,6 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"),
|
||||
brokenQuery);
|
||||
brokenRequest.setIndices(indices);
|
||||
brokenRequest.setTypes(types);
|
||||
specifications.add(brokenRequest);
|
||||
|
||||
RankEvalSpec task = new RankEvalSpec(specifications, new Precision());
|
||||
|
|
@ -173,7 +167,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
private static List<RatedDocument> createRelevant(String... docs) {
|
||||
List<RatedDocument> relevant = new ArrayList<>();
|
||||
for (String doc : docs) {
|
||||
relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal()));
|
||||
relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal()));
|
||||
}
|
||||
return relevant;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -78,18 +78,15 @@ public class RankEvalSpecTests extends ESTestCase {
|
|||
}
|
||||
|
||||
templates = new HashSet<>();
|
||||
templates.add(new ScriptWithId("templateId",
|
||||
new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, script, params)));
|
||||
templates.add(new ScriptWithId("templateId", new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, script, params)));
|
||||
|
||||
Map<String, Object> templateParams = new HashMap<>();
|
||||
templateParams.put("key", "value");
|
||||
RatedRequest ratedRequest = new RatedRequest("id",
|
||||
Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams,
|
||||
RatedRequest ratedRequest = new RatedRequest("id", Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams,
|
||||
"templateId");
|
||||
ratedRequests = Arrays.asList(ratedRequest);
|
||||
} else {
|
||||
RatedRequest ratedRequest = new RatedRequest("id",
|
||||
Arrays.asList(RatedDocumentTests.createRatedDocument()),
|
||||
RatedRequest ratedRequest = new RatedRequest("id", Arrays.asList(RatedDocumentTests.createRatedDocument()),
|
||||
new SearchSourceBuilder());
|
||||
ratedRequests = Arrays.asList(ratedRequest);
|
||||
}
|
||||
|
|
@ -101,15 +98,11 @@ public class RankEvalSpecTests extends ESTestCase {
|
|||
public void testXContentRoundtrip() throws IOException {
|
||||
RankEvalSpec testItem = createTestItem();
|
||||
|
||||
XContentBuilder shuffled = shuffleXContent(
|
||||
testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS));
|
||||
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS));
|
||||
try (XContentParser parser = createParser(JsonXContent.jsonXContent, shuffled.bytes())) {
|
||||
|
||||
RankEvalSpec parsedItem = RankEvalSpec.parse(parser);
|
||||
// IRL these come from URL parameters - see RestRankEvalAction
|
||||
// TODO Do we still need this?
|
||||
// parsedItem.getRatedRequests().stream().forEach(e ->
|
||||
// {e.setIndices(indices); e.setTypes(types);});
|
||||
assertNotSame(testItem, parsedItem);
|
||||
assertEquals(testItem, parsedItem);
|
||||
assertEquals(testItem.hashCode(), parsedItem.hashCode());
|
||||
|
|
@ -120,17 +113,13 @@ public class RankEvalSpecTests extends ESTestCase {
|
|||
RankEvalSpec original = createTestItem();
|
||||
|
||||
List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class,
|
||||
MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class,
|
||||
Precision.NAME, Precision::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class,
|
||||
DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class,
|
||||
ReciprocalRank.NAME, ReciprocalRank::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME,
|
||||
DiscountedCumulativeGain::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new));
|
||||
|
||||
RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new,
|
||||
new NamedWriteableRegistry(namedWriteables));
|
||||
RankEvalSpec deserialized = RankEvalTestHelper.copy(original, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables));
|
||||
assertEquals(deserialized, original);
|
||||
assertEquals(deserialized.hashCode(), original.hashCode());
|
||||
assertNotSame(deserialized, original);
|
||||
|
|
@ -140,20 +129,15 @@ public class RankEvalSpecTests extends ESTestCase {
|
|||
RankEvalSpec testItem = createTestItem();
|
||||
|
||||
List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class,
|
||||
MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class,
|
||||
Precision.NAME, Precision::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class,
|
||||
DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class,
|
||||
ReciprocalRank.NAME, ReciprocalRank::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, Precision.NAME, Precision::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, DiscountedCumulativeGain.NAME,
|
||||
DiscountedCumulativeGain::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(RankedListQualityMetric.class, ReciprocalRank.NAME, ReciprocalRank::new));
|
||||
|
||||
RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new,
|
||||
new NamedWriteableRegistry(namedWriteables));
|
||||
RankEvalSpec mutant = RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables));
|
||||
RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(mutant),
|
||||
RankEvalTestHelper.copy(testItem, RankEvalSpec::new,
|
||||
new NamedWriteableRegistry(namedWriteables)));
|
||||
RankEvalTestHelper.copy(testItem, RankEvalSpec::new, new NamedWriteableRegistry(namedWriteables)));
|
||||
}
|
||||
|
||||
private static RankEvalSpec mutateTestItem(RankEvalSpec mutant) {
|
||||
|
|
@ -164,8 +148,7 @@ public class RankEvalSpecTests extends ESTestCase {
|
|||
int mutate = randomIntBetween(0, 2);
|
||||
switch (mutate) {
|
||||
case 0:
|
||||
RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(),
|
||||
new ArrayList<>(), true);
|
||||
RatedRequest request = RatedRequestsTests.createTestItem(new ArrayList<>(), true);
|
||||
ratedRequests.add(request);
|
||||
break;
|
||||
case 1:
|
||||
|
|
@ -176,16 +159,7 @@ public class RankEvalSpecTests extends ESTestCase {
|
|||
}
|
||||
break;
|
||||
case 2:
|
||||
if (templates.size() > 0) {
|
||||
String mutatedTemplate = randomAlphaOfLength(10);
|
||||
templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate,
|
||||
new HashMap<>()));
|
||||
} else {
|
||||
String mutatedTemplate = randomValueOtherThanMany(templates::containsValue,
|
||||
() -> randomAlphaOfLength(10));
|
||||
templates.put("mutation", new Script(ScriptType.INLINE, "mustache", mutatedTemplate,
|
||||
new HashMap<>()));
|
||||
}
|
||||
templates.put("mutation", new Script(ScriptType.INLINE, "mustache", randomAlphaOfLength(10), new HashMap<>()));
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException("Requested to modify more than available parameters.");
|
||||
|
|
@ -202,28 +176,24 @@ public class RankEvalSpecTests extends ESTestCase {
|
|||
|
||||
public void testMissingRatedRequestsFailsParsing() {
|
||||
RankedListQualityMetric metric = new Precision();
|
||||
expectThrows(IllegalStateException.class,
|
||||
() -> new RankEvalSpec(new ArrayList<>(), metric));
|
||||
expectThrows(IllegalStateException.class, () -> new RankEvalSpec(new ArrayList<>(), metric));
|
||||
expectThrows(IllegalStateException.class, () -> new RankEvalSpec(null, metric));
|
||||
}
|
||||
|
||||
public void testMissingMetricFailsParsing() {
|
||||
List<String> strings = Arrays.asList("value");
|
||||
List<RatedRequest> ratedRequests = randomList(
|
||||
() -> RatedRequestsTests.createTestItem(strings, strings, randomBoolean()));
|
||||
List<RatedRequest> ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(strings, randomBoolean()));
|
||||
expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, null));
|
||||
}
|
||||
|
||||
public void testMissingTemplateAndSearchRequestFailsParsing() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("key", "value");
|
||||
|
||||
RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId");
|
||||
List<RatedRequest> ratedRequests = Arrays.asList(request);
|
||||
|
||||
expectThrows(IllegalStateException.class,
|
||||
() -> new RankEvalSpec(ratedRequests, new Precision()));
|
||||
expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new Precision()));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import static org.junit.Assert.assertNotSame;
|
|||
import static org.junit.Assert.assertThat;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
// TODO replace by infra from ESTestCase
|
||||
public class RankEvalTestHelper {
|
||||
|
||||
public static <T> void testHashCodeAndEquals(T testItem, T mutation, T secondCopy) {
|
||||
|
|
|
|||
|
|
@ -32,18 +32,16 @@ public class RatedDocumentTests extends ESTestCase {
|
|||
|
||||
public static RatedDocument createRatedDocument() {
|
||||
String index = randomAlphaOfLength(10);
|
||||
String type = randomAlphaOfLength(10);
|
||||
String docId = randomAlphaOfLength(10);
|
||||
int rating = randomInt();
|
||||
|
||||
return new RatedDocument(index, type, docId, rating);
|
||||
return new RatedDocument(index, docId, rating);
|
||||
}
|
||||
|
||||
public void testXContentParsing() throws IOException {
|
||||
RatedDocument testItem = createRatedDocument();
|
||||
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
|
||||
XContentBuilder shuffled = shuffleXContent(
|
||||
testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
try (XContentParser itemParser = createParser(shuffled)) {
|
||||
RatedDocument parsedItem = RatedDocument.fromXContent(itemParser);
|
||||
assertNotSame(testItem, parsedItem);
|
||||
|
|
@ -62,29 +60,22 @@ public class RatedDocumentTests extends ESTestCase {
|
|||
|
||||
public void testEqualsAndHash() throws IOException {
|
||||
RatedDocument testItem = createRatedDocument();
|
||||
RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem),
|
||||
RankEvalTestHelper.copy(testItem, RatedDocument::new));
|
||||
RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem), RankEvalTestHelper.copy(testItem, RatedDocument::new));
|
||||
}
|
||||
|
||||
public void testInvalidParsing() {
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedDocument(null, "abc", "abc", 10));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", "abc", 10));
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedDocument("abc", null, "abc", 10));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "", "abc", 10));
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedDocument("abc", "abc", null, 10));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "abc", "", 10));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedDocument(null, "abc", 10));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedDocument("", "abc", 10));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", "", 10));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedDocument("abc", null, 10));
|
||||
}
|
||||
|
||||
private static RatedDocument mutateTestItem(RatedDocument original) {
|
||||
int rating = original.getRating();
|
||||
String index = original.getIndex();
|
||||
String type = original.getType();
|
||||
String docId = original.getDocID();
|
||||
|
||||
switch (randomIntBetween(0, 3)) {
|
||||
switch (randomIntBetween(0, 2)) {
|
||||
case 0:
|
||||
rating = randomValueOtherThan(rating, () -> randomInt());
|
||||
break;
|
||||
|
|
@ -92,14 +83,11 @@ public class RatedDocumentTests extends ESTestCase {
|
|||
index = randomValueOtherThan(index, () -> randomAlphaOfLength(10));
|
||||
break;
|
||||
case 2:
|
||||
type = randomValueOtherThan(type, () -> randomAlphaOfLength(10));
|
||||
break;
|
||||
case 3:
|
||||
docId = randomValueOtherThan(docId, () -> randomAlphaOfLength(10));
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException("The test should only allow two parameters mutated");
|
||||
}
|
||||
return new RatedDocument(index, type, docId, rating);
|
||||
return new RatedDocument(index, docId, rating);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -57,9 +57,9 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
*/
|
||||
@BeforeClass
|
||||
public static void init() {
|
||||
xContentRegistry = new NamedXContentRegistry(Stream.of(
|
||||
new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream())
|
||||
.flatMap(Function.identity()).collect(toList()));
|
||||
xContentRegistry = new NamedXContentRegistry(
|
||||
Stream.of(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream()).flatMap(Function.identity())
|
||||
.collect(toList()));
|
||||
}
|
||||
|
||||
@AfterClass
|
||||
|
|
@ -72,8 +72,7 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
return xContentRegistry;
|
||||
}
|
||||
|
||||
public static RatedRequest createTestItem(List<String> indices, List<String> types,
|
||||
boolean forceRequest) {
|
||||
public static RatedRequest createTestItem(List<String> indices, boolean forceRequest) {
|
||||
String requestId = randomAlphaOfLength(50);
|
||||
|
||||
List<RatedDocument> ratedDocs = new ArrayList<>();
|
||||
|
|
@ -105,12 +104,10 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
if (params.size() == 0) {
|
||||
ratedRequest = new RatedRequest(requestId, ratedDocs, testRequest);
|
||||
ratedRequest.setIndices(indices);
|
||||
ratedRequest.setTypes(types);
|
||||
ratedRequest.setSummaryFields(summaryFields);
|
||||
} else {
|
||||
ratedRequest = new RatedRequest(requestId, ratedDocs, params, randomAlphaOfLength(5));
|
||||
ratedRequest.setIndices(indices);
|
||||
ratedRequest.setTypes(types);
|
||||
ratedRequest.setSummaryFields(summaryFields);
|
||||
}
|
||||
return ratedRequest;
|
||||
|
|
@ -123,25 +120,14 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
indices.add(randomAlphaOfLengthBetween(0, 50));
|
||||
}
|
||||
|
||||
List<String> types = new ArrayList<>();
|
||||
size = randomIntBetween(0, 20);
|
||||
for (int i = 0; i < size; i++) {
|
||||
types.add(randomAlphaOfLengthBetween(0, 50));
|
||||
}
|
||||
|
||||
RatedRequest testItem = createTestItem(indices, types, randomBoolean());
|
||||
RatedRequest testItem = createTestItem(indices, randomBoolean());
|
||||
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
|
||||
XContentBuilder shuffled = shuffleXContent(
|
||||
testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
try (XContentParser itemParser = createParser(shuffled)) {
|
||||
itemParser.nextToken();
|
||||
|
||||
RatedRequest parsedItem = RatedRequest.fromXContent(itemParser);
|
||||
parsedItem.setIndices(indices); // IRL these come from URL
|
||||
// parameters - see
|
||||
// RestRankEvalAction
|
||||
parsedItem.setTypes(types); // IRL these come from URL parameters -
|
||||
// see RestRankEvalAction
|
||||
parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction
|
||||
assertNotSame(testItem, parsedItem);
|
||||
assertEquals(testItem, parsedItem);
|
||||
assertEquals(testItem.hashCode(), parsedItem.hashCode());
|
||||
|
|
@ -155,20 +141,12 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
indices.add(randomAlphaOfLengthBetween(0, 50));
|
||||
}
|
||||
|
||||
List<String> types = new ArrayList<>();
|
||||
size = randomIntBetween(0, 20);
|
||||
for (int i = 0; i < size; i++) {
|
||||
types.add(randomAlphaOfLengthBetween(0, 50));
|
||||
}
|
||||
|
||||
RatedRequest original = createTestItem(indices, types, randomBoolean());
|
||||
RatedRequest original = createTestItem(indices, randomBoolean());
|
||||
|
||||
List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class,
|
||||
MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
|
||||
RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new,
|
||||
new NamedWriteableRegistry(namedWriteables));
|
||||
RatedRequest deserialized = RankEvalTestHelper.copy(original, RatedRequest::new, new NamedWriteableRegistry(namedWriteables));
|
||||
assertEquals(deserialized, original);
|
||||
assertEquals(deserialized.hashCode(), original.hashCode());
|
||||
assertNotSame(deserialized, original);
|
||||
|
|
@ -181,21 +159,13 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
indices.add(randomAlphaOfLengthBetween(0, 50));
|
||||
}
|
||||
|
||||
List<String> types = new ArrayList<>();
|
||||
size = randomIntBetween(0, 20);
|
||||
for (int i = 0; i < size; i++) {
|
||||
types.add(randomAlphaOfLengthBetween(0, 50));
|
||||
}
|
||||
|
||||
RatedRequest testItem = createTestItem(indices, types, randomBoolean());
|
||||
RatedRequest testItem = createTestItem(indices, randomBoolean());
|
||||
|
||||
List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class,
|
||||
MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
|
||||
|
||||
RankEvalTestHelper.testHashCodeAndEquals(testItem, mutateTestItem(testItem),
|
||||
RankEvalTestHelper.copy(testItem, RatedRequest::new,
|
||||
new NamedWriteableRegistry(namedWriteables)));
|
||||
RankEvalTestHelper.copy(testItem, RatedRequest::new, new NamedWriteableRegistry(namedWriteables)));
|
||||
}
|
||||
|
||||
private static RatedRequest mutateTestItem(RatedRequest original) {
|
||||
|
|
@ -203,12 +173,11 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
SearchSourceBuilder testRequest = original.getTestRequest();
|
||||
List<RatedDocument> ratedDocs = original.getRatedDocs();
|
||||
List<String> indices = original.getIndices();
|
||||
List<String> types = original.getTypes();
|
||||
Map<String, Object> params = original.getParams();
|
||||
List<String> summaryFields = original.getSummaryFields();
|
||||
String templateId = original.getTemplateId();
|
||||
|
||||
int mutate = randomIntBetween(0, 5);
|
||||
int mutate = randomIntBetween(0, 4);
|
||||
switch (mutate) {
|
||||
case 0:
|
||||
id = randomValueOtherThan(id, () -> randomAlphaOfLength(10));
|
||||
|
|
@ -231,120 +200,95 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
}
|
||||
break;
|
||||
case 2:
|
||||
ratedDocs = Arrays.asList(randomValueOtherThanMany(ratedDocs::contains,
|
||||
() -> RatedDocumentTests.createRatedDocument()));
|
||||
ratedDocs = Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument()));
|
||||
break;
|
||||
case 3:
|
||||
indices = Arrays.asList(
|
||||
randomValueOtherThanMany(indices::contains, () -> randomAlphaOfLength(10)));
|
||||
indices = Arrays.asList(randomValueOtherThanMany(indices::contains, () -> randomAlphaOfLength(10)));
|
||||
break;
|
||||
case 4:
|
||||
types = Arrays.asList(
|
||||
randomValueOtherThanMany(types::contains, () -> randomAlphaOfLength(10)));
|
||||
break;
|
||||
case 5:
|
||||
summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains,
|
||||
() -> randomAlphaOfLength(10)));
|
||||
summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAlphaOfLength(10)));
|
||||
break;
|
||||
default:
|
||||
throw new IllegalStateException("Requested to modify more than available parameters.");
|
||||
}
|
||||
|
||||
RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params,
|
||||
templateId);
|
||||
RatedRequest ratedRequest = new RatedRequest(id, ratedDocs, testRequest, params, templateId);
|
||||
ratedRequest.setIndices(indices);
|
||||
ratedRequest.setTypes(types);
|
||||
ratedRequest.setSummaryFields(summaryFields);
|
||||
|
||||
return ratedRequest;
|
||||
}
|
||||
|
||||
public void testDuplicateRatedDocThrowsException() {
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(
|
||||
new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1),
|
||||
new RatedDocument(new DocumentKey("index1", "type1", "id1"), 5));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1),
|
||||
new RatedDocument(new DocumentKey("index1", "id1"), 5));
|
||||
|
||||
// search request set, no summary fields
|
||||
IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder()));
|
||||
assertEquals(
|
||||
"Found duplicate rated document key [{\"_index\":\"index1\","
|
||||
+ "\"_type\":\"type1\",\"_id\":\"id1\"}]",
|
||||
assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}]",
|
||||
ex.getMessage());
|
||||
// templated path, no summary fields
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("key", "value");
|
||||
ex = expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, params, "templateId"));
|
||||
assertEquals(
|
||||
"Found duplicate rated document key [{\"_index\":\"index1\","
|
||||
+ "\"_type\":\"type1\",\"_id\":\"id1\"}]",
|
||||
ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, params, "templateId"));
|
||||
assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}]",
|
||||
ex.getMessage());
|
||||
}
|
||||
|
||||
public void testNullSummaryFieldsTreatment() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder());
|
||||
expectThrows(IllegalArgumentException.class, () -> request.setSummaryFields(null));
|
||||
}
|
||||
|
||||
public void testNullParamsTreatment() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null,
|
||||
null);
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, null);
|
||||
assertNotNull(request.getParams());
|
||||
}
|
||||
|
||||
public void testSettingParamsAndRequestThrows() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("key", "value");
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), params, null));
|
||||
}
|
||||
|
||||
public void testSettingNeitherParamsNorRequestThrows() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, null, null));
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId"));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, new HashMap<>(), "templateId"));
|
||||
}
|
||||
|
||||
public void testSettingParamsWithoutTemplateIdThrows() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
Map<String, Object> params = new HashMap<>();
|
||||
params.put("key", "value");
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, null, params, null));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, params, null));
|
||||
}
|
||||
|
||||
public void testSettingTemplateIdAndRequestThrows() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs,
|
||||
new SearchSourceBuilder(), null, "templateId"));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, new SearchSourceBuilder(), null, "templateId"));
|
||||
}
|
||||
|
||||
public void testSettingTemplateIdNoParamsThrows() {
|
||||
List<RatedDocument> ratedDocs = Arrays
|
||||
.asList(new RatedDocument(new DocumentKey("index1", "type1", "id1"), 1));
|
||||
expectThrows(IllegalArgumentException.class,
|
||||
() -> new RatedRequest("id", ratedDocs, null, null, "templateId"));
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument(new DocumentKey("index1", "id1"), 1));
|
||||
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null, "templateId"));
|
||||
}
|
||||
|
||||
/**
|
||||
* test that modifying the order of index/type/docId to make sure it doesn't
|
||||
* test that modifying the order of index/docId to make sure it doesn't
|
||||
* matter for parsing xContent
|
||||
*/
|
||||
public void testParseFromXContent() throws IOException {
|
||||
String querySpecString = " {\n" + " \"id\": \"my_qa_query\",\n" + " \"request\": {\n"
|
||||
+ " \"query\": {\n" + " \"bool\": {\n"
|
||||
String querySpecString = " {\n"
|
||||
+ " \"id\": \"my_qa_query\",\n"
|
||||
+ " \"request\": {\n"
|
||||
+ " \"query\": {\n"
|
||||
+ " \"bool\": {\n"
|
||||
+ " \"must\": [\n"
|
||||
+ " {\"match\": {\"beverage\": \"coffee\"}},\n"
|
||||
+ " {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n"
|
||||
|
|
@ -352,16 +296,15 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
+ " {\"value\": \"morning\",\"boost\": 2}}},\n"
|
||||
+ " {\"term\": {\"ip_location\": "
|
||||
+ " {\"value\": \"ams\",\"boost\": 10}}}]}\n"
|
||||
+ " },\n" + " \"size\": 10\n" + " },\n"
|
||||
+ " },\n"
|
||||
+ " \"size\": 10\n"
|
||||
+ " },\n"
|
||||
+ " \"summary_fields\" : [\"title\"],\n"
|
||||
+ " \"ratings\": [\n"
|
||||
+ " {\"_index\": \"test\", \"_type\": \"testtype\", "
|
||||
+ " \"_id\": \"1\", \"rating\" : 1 }, "
|
||||
+ " {\"_type\": \"testtype\", \"_index\": \"test\", "
|
||||
+ " \"_id\": \"2\", \"rating\" : 0 }, "
|
||||
+ " {\"_id\": \"3\", \"_index\": \"test\", "
|
||||
+ " \"_type\": \"testtype\", \"rating\" : 1 }]\n"
|
||||
+ "}";
|
||||
+ " {\"_index\": \"test\" , \"_id\": \"1\", \"rating\" : 1 },\n"
|
||||
+ " {\"_index\": \"test\", \"rating\" : 0, \"_id\": \"2\"},\n"
|
||||
+ " {\"_id\": \"3\", \"_index\": \"test\", \"rating\" : 1} ]"
|
||||
+ "}\n";
|
||||
try (XContentParser parser = createParser(JsonXContent.jsonXContent, querySpecString)) {
|
||||
RatedRequest specification = RatedRequest.fromXContent(parser);
|
||||
assertEquals("my_qa_query", specification.getId());
|
||||
|
|
@ -371,7 +314,6 @@ public class RatedRequestsTests extends ESTestCase {
|
|||
for (int i = 0; i < 3; i++) {
|
||||
assertEquals("" + (i + 1), ratedDocs.get(i).getDocID());
|
||||
assertEquals("test", ratedDocs.get(i).getIndex());
|
||||
assertEquals("testtype", ratedDocs.get(i).getType());
|
||||
if (i == 1) {
|
||||
assertEquals(0, ratedDocs.get(i).getRating());
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -45,75 +45,67 @@ public class ReciprocalRankTests extends ESTestCase {
|
|||
|
||||
int searchHits = randomIntBetween(1, 50);
|
||||
|
||||
SearchHit[] hits = createSearchHits(0, searchHits, "test", "type");
|
||||
SearchHit[] hits = createSearchHits(0, searchHits, "test");
|
||||
List<RatedDocument> ratedDocs = new ArrayList<>();
|
||||
int relevantAt = randomIntBetween(0, searchHits);
|
||||
for (int i = 0; i <= searchHits; i++) {
|
||||
if (i == relevantAt) {
|
||||
ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i),
|
||||
Rating.RELEVANT.ordinal()));
|
||||
ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.RELEVANT.ordinal()));
|
||||
} else {
|
||||
ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i),
|
||||
Rating.IRRELEVANT.ordinal()));
|
||||
ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.IRRELEVANT.ordinal()));
|
||||
}
|
||||
}
|
||||
|
||||
int rankAtFirstRelevant = relevantAt + 1;
|
||||
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
|
||||
assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE);
|
||||
assertEquals(rankAtFirstRelevant,
|
||||
((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
|
||||
assertEquals(rankAtFirstRelevant, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
|
||||
|
||||
// check that if we have fewer search hits than relevant doc position,
|
||||
// we don't find any result and get 0.0 quality level
|
||||
reciprocalRank = new ReciprocalRank();
|
||||
evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt),
|
||||
ratedDocs);
|
||||
evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs);
|
||||
assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
|
||||
}
|
||||
|
||||
public void testEvaluationOneRelevantInResults() {
|
||||
ReciprocalRank reciprocalRank = new ReciprocalRank();
|
||||
SearchHit[] hits = createSearchHits(0, 9, "test", "type");
|
||||
SearchHit[] hits = createSearchHits(0, 9, "test");
|
||||
List<RatedDocument> ratedDocs = new ArrayList<>();
|
||||
// mark one of the ten docs relevant
|
||||
int relevantAt = randomIntBetween(0, 9);
|
||||
for (int i = 0; i <= 20; i++) {
|
||||
if (i == relevantAt) {
|
||||
ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i),
|
||||
Rating.RELEVANT.ordinal()));
|
||||
ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.RELEVANT.ordinal()));
|
||||
} else {
|
||||
ratedDocs.add(new RatedDocument("test", "type", Integer.toString(i),
|
||||
Rating.IRRELEVANT.ordinal()));
|
||||
ratedDocs.add(new RatedDocument("test", Integer.toString(i), Rating.IRRELEVANT.ordinal()));
|
||||
}
|
||||
}
|
||||
|
||||
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
|
||||
assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE);
|
||||
assertEquals(relevantAt + 1,
|
||||
((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
|
||||
assertEquals(relevantAt + 1, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
|
||||
}
|
||||
|
||||
/**
|
||||
* test that the relevant rating threshold can be set to something larger
|
||||
* than 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant,
|
||||
* so first relevant doc has third ranking position, so RR should be 1/3
|
||||
* test that the relevant rating threshold can be set to something larger than
|
||||
* 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, so first
|
||||
* relevant doc has third ranking position, so RR should be 1/3
|
||||
*/
|
||||
public void testPrecisionAtFiveRelevanceThreshold() {
|
||||
List<RatedDocument> rated = new ArrayList<>();
|
||||
rated.add(new RatedDocument("test", "testtype", "0", 0));
|
||||
rated.add(new RatedDocument("test", "testtype", "1", 1));
|
||||
rated.add(new RatedDocument("test", "testtype", "2", 2));
|
||||
rated.add(new RatedDocument("test", "testtype", "3", 3));
|
||||
rated.add(new RatedDocument("test", "testtype", "4", 4));
|
||||
SearchHit[] hits = createSearchHits(0, 5, "test", "testtype");
|
||||
rated.add(new RatedDocument("test", "0", 0));
|
||||
rated.add(new RatedDocument("test", "1", 1));
|
||||
rated.add(new RatedDocument("test", "2", 2));
|
||||
rated.add(new RatedDocument("test", "3", 3));
|
||||
rated.add(new RatedDocument("test", "4", 4));
|
||||
SearchHit[] hits = createSearchHits(0, 5, "test");
|
||||
|
||||
ReciprocalRank reciprocalRank = new ReciprocalRank();
|
||||
reciprocalRank.setRelevantRatingThreshhold(2);
|
||||
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated);
|
||||
assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001);
|
||||
assertEquals(3,
|
||||
((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
|
||||
assertEquals(3, ((ReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
|
||||
}
|
||||
|
||||
public void testCombine() {
|
||||
|
|
@ -127,7 +119,7 @@ public class ReciprocalRankTests extends ESTestCase {
|
|||
|
||||
public void testEvaluationNoRelevantInResults() {
|
||||
ReciprocalRank reciprocalRank = new ReciprocalRank();
|
||||
SearchHit[] hits = createSearchHits(0, 9, "test", "type");
|
||||
SearchHit[] hits = createSearchHits(0, 9, "test");
|
||||
List<RatedDocument> ratedDocs = new ArrayList<>();
|
||||
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
|
||||
assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
|
||||
|
|
@ -136,8 +128,7 @@ public class ReciprocalRankTests extends ESTestCase {
|
|||
public void testXContentRoundtrip() throws IOException {
|
||||
ReciprocalRank testItem = createTestItem();
|
||||
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
|
||||
XContentBuilder shuffled = shuffleXContent(
|
||||
testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
|
||||
try (XContentParser itemParser = createParser(shuffled)) {
|
||||
itemParser.nextToken();
|
||||
itemParser.nextToken();
|
||||
|
|
@ -149,13 +140,13 @@ public class ReciprocalRankTests extends ESTestCase {
|
|||
}
|
||||
|
||||
/**
|
||||
* Create SearchHits for testing, starting from dociId 'from' up to docId
|
||||
* 'to'. The search hits index and type also need to be provided
|
||||
* Create SearchHits for testing, starting from dociId 'from' up to docId 'to'.
|
||||
* The search hits index also need to be provided
|
||||
*/
|
||||
private static SearchHit[] createSearchHits(int from, int to, String index, String type) {
|
||||
private static SearchHit[] createSearchHits(int from, int to, String index) {
|
||||
SearchHit[] hits = new SearchHit[to + 1 - from];
|
||||
for (int i = from; i <= to; i++) {
|
||||
hits[i] = new SearchHit(i, i + "", new Text(type), Collections.emptyMap());
|
||||
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap());
|
||||
hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null));
|
||||
}
|
||||
return hits;
|
||||
|
|
@ -185,8 +176,7 @@ public class ReciprocalRankTests extends ESTestCase {
|
|||
private static ReciprocalRank mutateTestItem(ReciprocalRank testItem) {
|
||||
int relevantThreshold = testItem.getRelevantRatingThreshold();
|
||||
ReciprocalRank rank = new ReciprocalRank();
|
||||
rank.setRelevantRatingThreshhold(
|
||||
randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)));
|
||||
rank.setRelevantRatingThreshhold(randomValueOtherThan(relevantThreshold, () -> randomIntBetween(0, 10)));
|
||||
return rank;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -46,14 +46,14 @@
|
|||
"id": "amsterdam_query",
|
||||
"request": { "query": { "match" : {"text" : "amsterdam" }}},
|
||||
"ratings": [
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0},
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1},
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc3", "rating": 1}]
|
||||
{"_index": "foo", "_id": "doc1", "rating": 0},
|
||||
{"_index": "foo", "_id": "doc2", "rating": 1},
|
||||
{"_index": "foo", "_id": "doc3", "rating": 1}]
|
||||
},
|
||||
{
|
||||
"id" : "berlin_query",
|
||||
"request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
|
||||
"ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}]
|
||||
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { "ignore_unlabeled" : true }}
|
||||
|
|
@ -61,7 +61,7 @@
|
|||
|
||||
- match: { rank_eval.quality_level: 1}
|
||||
- match: { rank_eval.details.amsterdam_query.quality_level: 1.0}
|
||||
- match: { rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]}
|
||||
- match: { rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_id": "doc4"}]}
|
||||
- match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}}
|
||||
|
||||
- length: { rank_eval.details.amsterdam_query.hits: 3}
|
||||
|
|
@ -73,7 +73,7 @@
|
|||
- is_false: rank_eval.details.amsterdam_query.hits.2.rating
|
||||
|
||||
- match: { rank_eval.details.berlin_query.quality_level: 1.0}
|
||||
- match: { rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc4"}]}
|
||||
- match: { rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_id": "doc4"}]}
|
||||
- match: { rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
|
||||
- length: { rank_eval.details.berlin_query.hits: 2}
|
||||
- match: { rank_eval.details.berlin_query.hits.0.hit._id: "doc1" }
|
||||
|
|
@ -130,13 +130,13 @@
|
|||
"id": "amsterdam_query",
|
||||
"request": { "query": { "match" : {"text" : "amsterdam" }}},
|
||||
# doc4 should be returned in third position, so reciprocal rank is 1/3
|
||||
"ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}]
|
||||
"ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
|
||||
},
|
||||
{
|
||||
"id" : "berlin_query",
|
||||
"request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
|
||||
# doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2
|
||||
"ratings": [{"_index": "foo", "_type": "bar", "_id": "doc4", "rating": 1}]
|
||||
"ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
|
||||
}
|
||||
],
|
||||
"metric" : { "reciprocal_rank": {} }
|
||||
|
|
@ -146,9 +146,9 @@
|
|||
- match: {rank_eval.quality_level: 0.41666666666666663}
|
||||
- match: {rank_eval.details.amsterdam_query.quality_level: 0.3333333333333333}
|
||||
- match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": 3}}
|
||||
- match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc2"},
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc3"} ]}
|
||||
- match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_id": "doc2"},
|
||||
{"_index": "foo", "_id": "doc3"} ]}
|
||||
- match: {rank_eval.details.berlin_query.quality_level: 0.5}
|
||||
- match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}}
|
||||
- match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_type": "bar", "_id": "doc1"}]}
|
||||
- match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_id": "doc1"}]}
|
||||
|
||||
|
|
|
|||
|
|
@ -54,12 +54,12 @@
|
|||
"id": "dcg_query",
|
||||
"request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] },
|
||||
"ratings": [
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}]
|
||||
{"_index" : "foo", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
|
||||
}
|
||||
],
|
||||
"metric" : { "dcg": {}}
|
||||
|
|
@ -79,12 +79,12 @@
|
|||
"id": "dcg_query_reverse",
|
||||
"request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] },
|
||||
"ratings": [
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}]
|
||||
{"_index" : "foo", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
|
||||
},
|
||||
],
|
||||
"metric" : { "dcg": { }}
|
||||
|
|
@ -104,23 +104,23 @@
|
|||
"id": "dcg_query",
|
||||
"request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] },
|
||||
"ratings": [
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}]
|
||||
{"_index" : "foo", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
|
||||
},
|
||||
{
|
||||
"id": "dcg_query_reverse",
|
||||
"request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] },
|
||||
"ratings": [
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_type" : "bar", "_id" : "doc6", "rating": 2}]
|
||||
{"_index" : "foo", "_id" : "doc1", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc2", "rating": 2},
|
||||
{"_index" : "foo", "_id" : "doc3", "rating": 3},
|
||||
{"_index" : "foo", "_id" : "doc4", "rating": 0},
|
||||
{"_index" : "foo", "_id" : "doc5", "rating": 1},
|
||||
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
|
||||
},
|
||||
],
|
||||
"metric" : { "dcg": { }}
|
||||
|
|
|
|||
|
|
@ -19,12 +19,12 @@
|
|||
"id": "amsterdam_query",
|
||||
"request": { "query": { "match_all" : { }}},
|
||||
"ratings": [
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}]
|
||||
{"_index": "foo", "_id": "doc1", "rating": 1}]
|
||||
},
|
||||
{
|
||||
"id" : "invalid_query",
|
||||
"request": { "query": { "range" : { "bar" : { "from" : "Basel", "time_zone": "+01:00" }}}},
|
||||
"ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}]
|
||||
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { "ignore_unlabeled" : true }}
|
||||
|
|
|
|||
|
|
@ -26,9 +26,9 @@ import org.elasticsearch.index.rankeval.RankEvalRequest;
|
|||
import org.elasticsearch.index.rankeval.RankEvalRequestBuilder;
|
||||
import org.elasticsearch.index.rankeval.RankEvalResponse;
|
||||
import org.elasticsearch.index.rankeval.RankEvalSpec;
|
||||
import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId;
|
||||
import org.elasticsearch.index.rankeval.RatedDocument;
|
||||
import org.elasticsearch.index.rankeval.RatedRequest;
|
||||
import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.script.Script;
|
||||
import org.elasticsearch.script.ScriptType;
|
||||
|
|
@ -82,7 +82,6 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase {
|
|||
|
||||
public void testPrecisionAtRequest() throws IOException {
|
||||
List<String> indices = Arrays.asList(new String[] { "test" });
|
||||
List<String> types = Arrays.asList(new String[] { "testtype" });
|
||||
|
||||
List<RatedRequest> specifications = new ArrayList<>();
|
||||
Map<String, Object> ams_params = new HashMap<>();
|
||||
|
|
@ -90,7 +89,6 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase {
|
|||
RatedRequest amsterdamRequest = new RatedRequest(
|
||||
"amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params, MATCH_TEMPLATE);
|
||||
amsterdamRequest.setIndices(indices);
|
||||
amsterdamRequest.setTypes(types);
|
||||
|
||||
specifications.add(amsterdamRequest);
|
||||
|
||||
|
|
@ -99,12 +97,11 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase {
|
|||
RatedRequest berlinRequest = new RatedRequest(
|
||||
"berlin_query", createRelevant("1"), berlin_params, MATCH_TEMPLATE);
|
||||
berlinRequest.setIndices(indices);
|
||||
berlinRequest.setTypes(types);
|
||||
specifications.add(berlinRequest);
|
||||
|
||||
Precision metric = new Precision();
|
||||
|
||||
ScriptWithId template =
|
||||
ScriptWithId template =
|
||||
new ScriptWithId(
|
||||
MATCH_TEMPLATE,
|
||||
new Script(
|
||||
|
|
@ -124,7 +121,7 @@ public class SmokeMultipleTemplatesIT extends ESIntegTestCase {
|
|||
private static List<RatedDocument> createRelevant(String... docs) {
|
||||
List<RatedDocument> relevant = new ArrayList<>();
|
||||
for (String doc : docs) {
|
||||
relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal()));
|
||||
relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal()));
|
||||
}
|
||||
return relevant;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -41,22 +41,22 @@
|
|||
- do:
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"inline": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ],
|
||||
"templates": [ { "id": "match", "template": {"source": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": [
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0},
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1},
|
||||
{"_index": "foo", "_type": "bar", "_id": "doc3", "rating": 1}]
|
||||
{"_index": "foo", "_id": "doc1", "rating": 0},
|
||||
{"_index": "foo", "_id": "doc2", "rating": 1},
|
||||
{"_index": "foo", "_id": "doc3", "rating": 1}]
|
||||
},
|
||||
{
|
||||
"id" : "berlin_query",
|
||||
"params": { "query_string": "berlin" },
|
||||
"template_id": "match",
|
||||
"ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}]
|
||||
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
|
|
|
|||
Loading…
Reference in New Issue