honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'rankeval' 

This commit adds a new module that provides an endpoint that can be used to
evaluate search ranking results.

Closes #19195
This commit is contained in:
Christoph Büscher 2017-12-14 15:33:59 +01:00
parent 10d2667498 5406a9f30d
commit bb14b8f7c5
48 changed files with 5460 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
build.gradle
View File

@ -196,6 +196,7 @@ subprojects {
"org.elasticsearch.plugin:parent-join-client:${version}": ':modules:parent-join',
"org.elasticsearch.plugin:aggs-matrix-stats-client:${version}": ':modules:aggs-matrix-stats',
"org.elasticsearch.plugin:percolator-client:${version}": ':modules:percolator',
"org.elasticsearch.plugin:rank-eval-client:${version}": ':modules:rank-eval',
]
for (final Version version : versionCollection.versionsIndexCompatibleWithCurrent) {

3
client/rest-high-level/build.gradle
View File

@ -39,6 +39,7 @@ dependencies {
compile "org.elasticsearch.client:elasticsearch-rest-client:${version}"
compile "org.elasticsearch.plugin:parent-join-client:${version}"
compile "org.elasticsearch.plugin:aggs-matrix-stats-client:${version}"
compile "org.elasticsearch.plugin:rank-eval-client:${version}"
testCompile "org.elasticsearch.client:test:${version}"
testCompile "org.elasticsearch.test:framework:${version}"
@ -60,4 +61,4 @@ forbiddenApisMain {
// specified
signaturesURLs += [PrecommitTasks.getResource('/forbidden/http-signatures.txt')]
signaturesURLs += [file('src/main/resources/forbidden/rest-high-level-signatures.txt').toURI().toURL()]
}
}

40
client/rest-high-level/src/test/java/org/elasticsearch/client/RestHighLevelClientTests.java
View File

@ -21,20 +21,6 @@ package org.elasticsearch.client;
import com.fasterxml.jackson.core.JsonParseException;
import org.elasticsearch.Build;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.main.MainRequest;
import org.elasticsearch.action.main.MainResponse;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.ClearScrollResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchResponseSections;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
@ -49,6 +35,20 @@ import org.apache.http.message.BasicHttpResponse;
import org.apache.http.message.BasicRequestLine;
import org.apache.http.message.BasicStatusLine;
import org.apache.http.nio.entity.NStringEntity;
import org.elasticsearch.Build;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.Version;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.main.MainRequest;
import org.elasticsearch.action.main.MainResponse;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.ClearScrollResponse;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchResponseSections;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.search.ShardSearchFailure;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@ -57,6 +57,10 @@ import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.cbor.CborXContent;
import org.elasticsearch.common.xcontent.smile.SmileXContent;
import org.elasticsearch.index.rankeval.DiscountedCumulativeGain;
import org.elasticsearch.index.rankeval.EvaluationMetric;
import org.elasticsearch.index.rankeval.MeanReciprocalRank;
import org.elasticsearch.index.rankeval.PrecisionAtK;
import org.elasticsearch.join.aggregations.ChildrenAggregationBuilder;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.search.SearchHits;
@ -648,7 +652,7 @@ public class RestHighLevelClientTests extends ESTestCase {
public void testProvidedNamedXContents() {
List<NamedXContentRegistry.Entry> namedXContents = RestHighLevelClient.getProvidedNamedXContents();
assertEquals(2, namedXContents.size());
assertEquals(5, namedXContents.size());
Map<Class<?>, Integer> categories = new HashMap<>();
List<String> names = new ArrayList<>();
for (NamedXContentRegistry.Entry namedXContent : namedXContents) {
@ -658,10 +662,14 @@ public class RestHighLevelClientTests extends ESTestCase {
categories.put(namedXContent.categoryClass, counter + 1);
}
}
assertEquals(1, categories.size());
assertEquals(2, categories.size());
assertEquals(Integer.valueOf(2), categories.get(Aggregation.class));
assertTrue(names.contains(ChildrenAggregationBuilder.NAME));
assertTrue(names.contains(MatrixStatsAggregationBuilder.NAME));
assertEquals(Integer.valueOf(3), categories.get(EvaluationMetric.class));
assertTrue(names.contains(PrecisionAtK.NAME));
assertTrue(names.contains(DiscountedCumulativeGain.NAME));
assertTrue(names.contains(MeanReciprocalRank.NAME));
}
private static class TrackingActionListener implements ActionListener<Integer> {

3
client/transport/build.gradle
View File

@ -32,6 +32,7 @@ dependencies {
compile "org.elasticsearch.plugin:lang-mustache-client:${version}"
compile "org.elasticsearch.plugin:percolator-client:${version}"
compile "org.elasticsearch.plugin:parent-join-client:${version}"
compile "org.elasticsearch.plugin:rank-eval-client:${version}"
testCompile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
testCompile "junit:junit:${versions.junit}"
testCompile "org.hamcrest:hamcrest-all:${versions.hamcrest}"
@ -54,4 +55,4 @@ namingConventions {
testClass = 'com.carrotsearch.randomizedtesting.RandomizedTest'
//we don't have integration tests
skipIntegTestInDisguise = true
}
}

2
docs/Versions.asciidoc
View File

@ -37,6 +37,7 @@ ifeval::["{release-state}"=="unreleased"]
:parent-join-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}-SNAPSHOT
:percolator-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}-SNAPSHOT
:matrixstats-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/aggs-matrix-stats-client/{version}-SNAPSHOT
:rank-eval-client-javadoc: https://snapshots.elastic.co/javadoc/org/elasticsearch/plugin/rank-eval-client/{version}-SNAPSHOT
endif::[]
ifeval::["{release-state}"!="unreleased"]
@ -49,6 +50,7 @@ ifeval::["{release-state}"!="unreleased"]
:parent-join-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/parent-join-client/{version}
:percolator-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/percolator-client/{version}
:matrixstats-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/aggs-matrix-stats-client/{version}
:rank-eval-client-javadoc: https://artifacts.elastic.co/javadoc/org/elasticsearch/plugin/rank-eval-client/{version}
endif::[]
///////

2
docs/reference/search.asciidoc
View File

@ -162,3 +162,5 @@ include::search/explain.asciidoc[]
include::search/profile.asciidoc[]
include::search/field-caps.asciidoc[]
include::search/rank-eval.asciidoc[]

298
docs/reference/search/rank-eval.asciidoc Normal file
View File

@ -0,0 +1,298 @@
[[search-rank-eval]]
== Ranking Evaluation API
The ranking evaluation API allows to evaluate the quality of ranked search
results over a set of typical search queries. Given this set of queries and a
list or manually rated documents, the `_rank_eval` endpoint calculates and
returns typical information retrieval metrics like _mean reciprocal rank_,
_precision_ or _discounted cumulative gain_.
experimental[The ranking evaluation API is new and may change in non-backwards compatible ways in the future,
even on minor versions updates.]
=== Overview
Search quality evaluation starts with looking at the users of your search application, and the things that they are searching for.
Users have a specific _information need_, e.g. they are looking for gift in a web shop or want to book a flight for their next holiday.
They usually enters some search terms into a search box or some other web form.
All of this information, together with meta information about the user (e.g. the browser, location, earlier preferences etc...) then gets translated into a query to the underlying search system.
The challenge for search engineers is to tweak this translation process from user entries to a concrete query in such a way, that the search results contain the most relevant information with respect to the users information_need.
This can only be done if the search result quality is evaluated constantly across a representative test suite of typical user queries, so that improvements in the rankings for one particular query doesn't negatively effect the ranking for other types of queries.
In order to get started with search quality evaluation, three basic things are needed:
. a collection of documents you want to evaluate your query performance against, usually one or more indices
. a collection of typical search requests that users enter into your system
. a set of document ratings that judge the documents relevance with respect to a search request+
It is important to note that one set of document ratings is needed per test query, and that
the relevance judgements are based on the _information_need_ of the user that entered the query.
The ranking evaluation API provides a convenient way to use this information in a ranking evaluation request to calculate different search evaluation metrics. This gives a first estimation of your overall search quality and give you a measurement to optimize against when fine-tuning various aspect of the query generation in your application.
=== Ranking evaluation request structure
In its most basic form, a request to the `_rank_eval` endpoint has two sections:
[source,js]
-----------------------------
GET /my_index/_rank_eval
{
"requests": [ ... ], <1>
"metric": { <2>
"reciprocal_rank": { ... } <3>
}
}
------------------------------
// NOTCONSOLE
<1> a set of typical search requests, together with their provided ratings
<2> definition of the evaluation metric to calculate
<3> a specific metric and its parameters
The request section contains several search requests typical to your application, along with the document ratings for each particular search request, e.g.
[source,js]
-----------------------------
"requests": [
{
"id": "amsterdam_query", <1>
"request": { <2>
"query": { "match": { "text": "amsterdam" }}
},
"ratings": [ <3>
{ "_index": "my_index", "_id": "doc1", "rating": 0 },
{ "_index": "my_index", "_id": "doc2", "rating": 3},
{ "_index": "my_index", "_id": "doc3", "rating": 1 }
]
},
{
"id": "berlin_query",
"request": {
"query": { "match": { "text": "berlin" }}
},
"ratings": [
{ "_index": "my_index", "_id": "doc1", "rating": 1 }
]
}
]
------------------------------
// NOTCONSOLE
<1> the search requests id, used to group result details later
<2> the query that is being evaluated
<3> a list of document ratings, each entry containing the documents `_index` and `_id` together with
the rating of the documents relevance with regards to this search request
A document `rating` can be any integer value that expresses the relevance of the document on a user defined scale. For some of the metrics, just giving a binary rating (e.g. `0` for irrelevant and `1` for relevant) will be sufficient, other metrics can use a more fine grained scale.
=== Template based ranking evaluation
As an alternative to having to provide a single query per test request, it is possible to specify query templates in the evaluation request and later refer to them. Queries with similar structure that only differ in their parameters don't have to be repeated all the time in the `requests` section this way. In typical search systems where user inputs usually get filled into a small set of query templates, this helps making the evaluation request more succinct.
[source,js]
--------------------------------
GET /my_index/_rank_eval
{
[...]
"templates": [
{
"id": "match_one_field_query", <1>
"template": { <2>
"inline": {
"query": {
"match": { "{{field}}": { "query": "{{query_string}}" }}
}
}
}
}
],
"requests": [
{
"id": "amsterdam_query"
"ratings": [ ... ],
"template_id": "match_one_field_query", <3>
"params": { <4>
"query_string": "amsterdam",
"field": "text"
}
},
[...]
}
--------------------------------
// NOTCONSOLE
<1> the template id
<2> the template definition to use
<3> a reference to a previously defined temlate
<4> the parameters to use to fill the template
=== Available evaluation metrics
The `metric` section determines which of the available evaluation metrics is going to be used.
Currently, the following metrics are supported:
==== Precision at K (P@k)
This metric measures the number of relevant results in the top k search results. Its a form of the well known https://en.wikipedia.org/wiki/Information_retrieval#Precision[Precision] metric that only looks at the top k documents. It is the fraction of relevant documents in those first k
search. A precision at 10 (P@10) value of 0.6 then means six out of the 10 top hits are relevant with respect to the users information need.
P@k works well as a simple evaluation metric that has the benefit of being easy to understand and explain.
Documents in the collection need to be rated either as relevant or irrelevant with respect to the current query.
P@k does not take into account where in the top k results the relevant documents occur, so a ranking of ten results that
contains one relevant result in position 10 is equally good as a ranking of ten results that contains one relevant result in position 1.
[source,js]
--------------------------------
GET /twitter/_rank_eval
{
"requests": [
{
"id": "JFK query",
"request": { "query": { "match_all": {}}},
"ratings": []
}],
"metric": {
"precision": {
"relevant_rating_threshold": 1,
"ignore_unlabeled": false
}
}
}
--------------------------------
// CONSOLE
// TEST[setup:twitter]
The `precision` metric takes the following optional parameters
[cols="<,<",options="header",]
|=======================================================================
|Parameter |Description
|`relevant_rating_threshold` |Sets the rating threshold above which documents are considered to be
"relevant". Defaults to `1`.
|`ignore_unlabeled` |controls how unlabeled documents in the search results are counted.
If set to 'true', unlabeled documents are ignored and neither count as relevant or irrelevant. Set to 'false' (the default), they are treated as irrelevant.
|=======================================================================
==== Mean reciprocal rank
For every query in the test suite, this metric calculates the reciprocal of the rank of the
first relevant document. For example finding the first relevant result
in position 3 means the reciprocal rank is 1/3. The reciprocal rank for each query
is averaged across all queries in the test suite to give the https://en.wikipedia.org/wiki/Mean_reciprocal_rank[mean reciprocal rank].
[source,js]
--------------------------------
GET /twitter/_rank_eval
{
"requests": [
{
"id": "JFK query",
"request": { "query": { "match_all": {}}},
"ratings": []
}],
"metric": {
"mean_reciprocal_rank": {
"relevant_rating_threshold" : 1
}
}
}
--------------------------------
// CONSOLE
// TEST[setup:twitter]
The `mean_reciprocal_rank` metric takes the following optional parameters
[cols="<,<",options="header",]
|=======================================================================
|Parameter |Description
|`relevant_rating_threshold` |Sets the rating threshold above which documents are considered to be
"relevant". Defaults to `1`.
|=======================================================================
==== Discounted cumulative gain (DCG)
In contrast to the two metrics above, https://en.wikipedia.org/wiki/Discounted_cumulative_gain[discounted cumulative gain] takes both, the rank and the rating of the search results, into account.
The assumption is that highly relevant documents are more useful for the user when appearing at the top of the result list. Therefore, the DCG formula reduces the contribution that high ratings for documents on lower search ranks have on the overall DCG metric.
[source,js]
--------------------------------
GET /twitter/_rank_eval
{
"requests": [
{
"id": "JFK query",
"request": { "query": { "match_all": {}}},
"ratings": []
}],
"metric": {
"dcg": {
"normalize": false
}
}
}
--------------------------------
// CONSOLE
// TEST[setup:twitter]
The `dcg` metric takes the following optional parameters:
[cols="<,<",options="header",]
|=======================================================================
|Parameter |Description
|`normalize` | If set to `true`, this metric will calculate the https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Normalized_DCG[Normalized DCG].
|=======================================================================
=== Response format
The response of the `_rank_eval` endpoint contains the overall calculated result for the defined quality metric,
a `details` section with a breakdown of results for each query in the test suite and an optional `failures` section
that shows potential errors of individual queries. The response has the following format:
[source,js]
--------------------------------
{
"rank_eval": {
"quality_level": 0.4, <1>
"details": {
"my_query_id1": { <2>
"quality_level": 0.6, <3>
"unknown_docs": [ <4>
{
"_index": "my_index",
"_id": "1960795"
}, [...]
],
"hits": [
{
"hit": { <5>
"_index": "my_index",
"_type": "page",
"_id": "1528558",
"_score": 7.0556192
},
"rating": 1
}, [...]
],
"metric_details": { <6>
"relevant_docs_retrieved": 6,
"docs_retrieved": 10
}
},
"my_query_id2 : { [...]}
},
"failures": { [...] }
}
}
--------------------------------
// NOTCONSOLE
<1> the overall evaluation quality calculated by the defined metric
<2> the `details` section contains one entry for every query in the original `requests` section, keyed by the search request id
<3> the `quality_level` in the `details` section shows the contribution of this query to the global quality score
<4> the `unknown_docs` section contains an `_index` and `_id` entry for each document in the search result for this
query that didn't have a ratings value. This can be used to ask the user to supply ratings for these documents
<5> the `hits` section shows a grouping of the search results with their supplied rating
<6> the `metric_details` give additional information about the calculated quality metric (e.g. how many of the retrieved
documents where relevant). The content varies for each metric but allows for better interpretation of the results

25
modules/rank-eval/build.gradle Normal file
View File

@ -0,0 +1,25 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
esplugin {
description 'The Rank Eval module adds APIs to evaluate ranking quality.'
classname 'org.elasticsearch.index.rankeval.RankEvalPlugin'
hasClientJar = true
}

217
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGain.java Normal file
View File

@ -0,0 +1,217 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchHit;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings;
/**
* Metric implementing Discounted Cumulative Gain.
* The `normalize` parameter can be set to calculate the normalized NDCG (set to <tt>false</tt> by default).<br>
* The optional `unknown_doc_rating` parameter can be used to specify a default rating for unlabeled documents.
* @see <a href="https://en.wikipedia.org/wiki/Discounted_cumulative_gain#Discounted_Cumulative_Gain">Discounted Cumulative Gain</a><br>
*/
public class DiscountedCumulativeGain implements EvaluationMetric {
/** If set to true, the dcg will be normalized (ndcg) */
private final boolean normalize;
/** the default search window size */
private static final int DEFAULT_K = 10;
/** the search window size */
private final int k;
/**
* Optional. If set, this will be the rating for docs that are unrated in the ranking evaluation request
*/
private final Integer unknownDocRating;
public static final String NAME = "dcg";
private static final double LOG2 = Math.log(2.0);
public DiscountedCumulativeGain() {
this(false, null, DEFAULT_K);
}
/**
* @param normalize
* If set to true, dcg will be normalized (ndcg) See
* https://en.wikipedia.org/wiki/Discounted_cumulative_gain
* @param unknownDocRating
* the rating for documents the user hasn't supplied an explicit
* rating for
* @param k the search window size all request use.
*/
public DiscountedCumulativeGain(boolean normalize, Integer unknownDocRating, int k) {
this.normalize = normalize;
this.unknownDocRating = unknownDocRating;
this.k = k;
}
DiscountedCumulativeGain(StreamInput in) throws IOException {
normalize = in.readBoolean();
unknownDocRating = in.readOptionalVInt();
k = in.readVInt();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeBoolean(normalize);
out.writeOptionalVInt(unknownDocRating);
out.writeVInt(k);
}
@Override
public String getWriteableName() {
return NAME;
}
boolean getNormalize() {
return this.normalize;
}
int getK() {
return this.k;
}
/**
* get the rating used for unrated documents
*/
public Integer getUnknownDocRating() {
return this.unknownDocRating;
}
@Override
public Optional<Integer> forcedSearchSize() {
return Optional.of(k);
}
@Override
public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
List<RatedDocument> ratedDocs) {
List<Integer> allRatings = ratedDocs.stream().mapToInt(RatedDocument::getRating).boxed()
.collect(Collectors.toList());
List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs);
List<Integer> ratingsInSearchHits = new ArrayList<>(ratedHits.size());
for (RatedSearchHit hit : ratedHits) {
// unknownDocRating might be null, which means it will be unrated docs are
// ignored in the dcg calculation
// we still need to add them as a placeholder so the rank of the subsequent
// ratings is correct
ratingsInSearchHits.add(hit.getRating().orElse(unknownDocRating));
}
double dcg = computeDCG(ratingsInSearchHits);
if (normalize) {
Collections.sort(allRatings, Comparator.nullsLast(Collections.reverseOrder()));
double idcg = computeDCG(
allRatings.subList(0, Math.min(ratingsInSearchHits.size(), allRatings.size())));
dcg = dcg / idcg;
}
EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, dcg);
evalQueryQuality.addHitsAndRatings(ratedHits);
return evalQueryQuality;
}
private static double computeDCG(List<Integer> ratings) {
int rank = 1;
double dcg = 0;
for (Integer rating : ratings) {
if (rating != null) {
dcg += (Math.pow(2, rating) - 1) / ((Math.log(rank + 1) / LOG2));
}
rank++;
}
return dcg;
}
private static final ParseField K_FIELD = new ParseField("k");
private static final ParseField NORMALIZE_FIELD = new ParseField("normalize");
private static final ParseField UNKNOWN_DOC_RATING_FIELD = new ParseField("unknown_doc_rating");
private static final ConstructingObjectParser<DiscountedCumulativeGain, Void> PARSER = new ConstructingObjectParser<>("dcg_at",
args -> {
Boolean normalized = (Boolean) args[0];
Integer optK = (Integer) args[2];
return new DiscountedCumulativeGain(normalized == null ? false : normalized, (Integer) args[1],
optK == null ? DEFAULT_K : optK);
});
static {
PARSER.declareBoolean(optionalConstructorArg(), NORMALIZE_FIELD);
PARSER.declareInt(optionalConstructorArg(), UNKNOWN_DOC_RATING_FIELD);
PARSER.declareInt(optionalConstructorArg(), K_FIELD);
}
public static DiscountedCumulativeGain fromXContent(XContentParser parser) {
return PARSER.apply(parser, null);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject(NAME);
builder.field(NORMALIZE_FIELD.getPreferredName(), this.normalize);
if (unknownDocRating != null) {
builder.field(UNKNOWN_DOC_RATING_FIELD.getPreferredName(), this.unknownDocRating);
}
builder.field(K_FIELD.getPreferredName(), this.k);
builder.endObject();
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
DiscountedCumulativeGain other = (DiscountedCumulativeGain) obj;
return Objects.equals(normalize, other.normalize)
&& Objects.equals(unknownDocRating, other.unknownDocRating)
&& Objects.equals(k, other.k);
}
@Override
public final int hashCode() {
return Objects.hash(normalize, unknownDocRating, k);
}
}

132
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java Normal file
View File

@ -0,0 +1,132 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Objects;;
/**
* Result of the evaluation metric calculation on one particular query alone.
*/
public class EvalQueryQuality implements ToXContent, Writeable {
private final String queryId;
private final double evaluationResult;
private MetricDetails optionalMetricDetails;
private final List<RatedSearchHit> ratedHits = new ArrayList<>();
public EvalQueryQuality(String id, double evaluationResult) {
this.queryId = id;
this.evaluationResult = evaluationResult;
}
public EvalQueryQuality(StreamInput in) throws IOException {
this(in.readString(), in.readDouble());
this.ratedHits.addAll(in.readList(RatedSearchHit::new));
this.optionalMetricDetails = in.readOptionalNamedWriteable(MetricDetails.class);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(queryId);
out.writeDouble(evaluationResult);
out.writeList(ratedHits);
out.writeOptionalNamedWriteable(this.optionalMetricDetails);
}
public String getId() {
return queryId;
}
public double getQualityLevel() {
return evaluationResult;
}
public void setMetricDetails(MetricDetails breakdown) {
this.optionalMetricDetails = breakdown;
}
public MetricDetails getMetricDetails() {
return this.optionalMetricDetails;
}
public void addHitsAndRatings(List<RatedSearchHit> hits) {
this.ratedHits.addAll(hits);
}
public List<RatedSearchHit> getHitsAndRatings() {
return this.ratedHits;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject(queryId);
builder.field("quality_level", this.evaluationResult);
builder.startArray("unknown_docs");
for (DocumentKey key : EvaluationMetric.filterUnknownDocuments(ratedHits)) {
builder.startObject();
builder.field(RatedDocument.INDEX_FIELD.getPreferredName(), key.getIndex());
builder.field(RatedDocument.DOC_ID_FIELD.getPreferredName(), key.getDocId());
builder.endObject();
}
builder.endArray();
builder.startArray("hits");
for (RatedSearchHit hit : ratedHits) {
hit.toXContent(builder, params);
}
builder.endArray();
if (optionalMetricDetails != null) {
builder.startObject("metric_details");
optionalMetricDetails.toXContent(builder, params);
builder.endObject();
}
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
EvalQueryQuality other = (EvalQueryQuality) obj;
return Objects.equals(queryId, other.queryId) &&
Objects.equals(evaluationResult, other.evaluationResult) &&
Objects.equals(ratedHits, other.ratedHits) &&
Objects.equals(optionalMetricDetails, other.optionalMetricDetails);
}
@Override
public final int hashCode() {
return Objects.hash(queryId, evaluationResult, ratedHits, optionalMetricDetails);
}
}

101
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvaluationMetric.java Normal file
View File

@ -0,0 +1,101 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
/**
* Implementations of {@link EvaluationMetric} need to provide a way to compute the quality metric for
* a result list returned by some search (@link {@link SearchHits}) and a list of rated documents.
*/
public interface EvaluationMetric extends ToXContent, NamedWriteable {
/**
* Returns a single metric representing the ranking quality of a set of returned
* documents wrt. to a set of document ids labeled as relevant for this search.
*
* @param taskId
* the id of the query for which the ranking is currently evaluated
* @param hits
* the result hits as returned by a search request
* @param ratedDocs
* the documents that were ranked by human annotators for this query
* case
* @return some metric representing the quality of the result hit list wrt. to
* relevant doc ids.
*/
EvalQueryQuality evaluate(String taskId, SearchHit[] hits, List<RatedDocument> ratedDocs);
/**
* join hits with rated documents using the joint _index/_id document key
*/
static List<RatedSearchHit> joinHitsWithRatings(SearchHit[] hits, List<RatedDocument> ratedDocs) {
Map<DocumentKey, RatedDocument> ratedDocumentMap = ratedDocs.stream()
.collect(Collectors.toMap(RatedDocument::getKey, item -> item));
List<RatedSearchHit> ratedSearchHits = new ArrayList<>(hits.length);
for (SearchHit hit : hits) {
DocumentKey key = new DocumentKey(hit.getIndex(), hit.getId());
RatedDocument ratedDoc = ratedDocumentMap.get(key);
if (ratedDoc != null) {
ratedSearchHits.add(new RatedSearchHit(hit, Optional.of(ratedDoc.getRating())));
} else {
ratedSearchHits.add(new RatedSearchHit(hit, Optional.empty()));
}
}
return ratedSearchHits;
}
/**
* filter @link {@link RatedSearchHit} that don't have a rating
*/
static List<DocumentKey> filterUnknownDocuments(List<RatedSearchHit> ratedHits) {
List<DocumentKey> unknownDocs = ratedHits.stream().filter(hit -> hit.getRating().isPresent() == false)
.map(hit -> new DocumentKey(hit.getSearchHit().getIndex(), hit.getSearchHit().getId())).collect(Collectors.toList());
return unknownDocs;
}
/**
* how evaluation metrics for particular search queries get combined for the overall evaluation score.
* Defaults to averaging over the partial results.
*/
default double combine(Collection<EvalQueryQuality> partialResults) {
return partialResults.stream().mapToDouble(EvalQueryQuality::getQualityLevel).sum() / partialResults.size();
}
/**
* Metrics can define a size of the search hits windows they want to retrieve by overwriting
* this method. The default implementation returns an empty optional.
* @return the number of search hits this metrics requests
*/
default Optional<Integer> forcedSearchSize() {
return Optional.empty();
}
}

233
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MeanReciprocalRank.java Normal file
View File

@ -0,0 +1,233 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchHit;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings;
/**
* Metric implementing Mean Reciprocal Rank (https://en.wikipedia.org/wiki/Mean_reciprocal_rank).<br>
* By default documents with a rating equal or bigger than 1 are considered to be "relevant" for the reciprocal
* rank calculation. This value can be changes using the relevant_rating_threshold` parameter.
*/
public class MeanReciprocalRank implements EvaluationMetric {
public static final String NAME = "mean_reciprocal_rank";
private static final int DEFAULT_RATING_THRESHOLD = 1;
private static final int DEFAULT_K = 10;
/** the search window size */
private final int k;
/** ratings equal or above this value will be considered relevant */
private final int relevantRatingThreshhold;
public MeanReciprocalRank() {
this(DEFAULT_RATING_THRESHOLD, DEFAULT_K);
}
MeanReciprocalRank(StreamInput in) throws IOException {
this.relevantRatingThreshhold = in.readVInt();
this.k = in.readVInt();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(this.relevantRatingThreshhold);
out.writeVInt(this.k);
}
/**
* Metric implementing Mean Reciprocal Rank (https://en.wikipedia.org/wiki/Mean_reciprocal_rank).<br>
* @param relevantRatingThreshold the rating value that a document needs to be regarded as "relevant". Defaults to 1.
* @param k the search window size all request use.
*/
public MeanReciprocalRank(int relevantRatingThreshold, int k) {
if (relevantRatingThreshold < 0) {
throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer.");
}
if (k <= 0) {
throw new IllegalArgumentException("Window size k must be positive.");
}
this.k = k;
this.relevantRatingThreshhold = relevantRatingThreshold;
}
int getK() {
return this.k;
}
@Override
public Optional<Integer> forcedSearchSize() {
return Optional.of(k);
}
@Override
public String getWriteableName() {
return NAME;
}
/**
* Return the rating threshold above which ratings are considered to be "relevant".
*/
public int getRelevantRatingThreshold() {
return relevantRatingThreshhold;
}
/**
* Compute ReciprocalRank based on provided relevant document IDs.
**/
@Override
public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
List<RatedDocument> ratedDocs) {
List<RatedSearchHit> ratedHits = joinHitsWithRatings(hits, ratedDocs);
int firstRelevant = -1;
int rank = 1;
for (RatedSearchHit hit : ratedHits) {
Optional<Integer> rating = hit.getRating();
if (rating.isPresent()) {
if (rating.get() >= this.relevantRatingThreshhold) {
firstRelevant = rank;
break;
}
}
rank++;
}
double reciprocalRank = (firstRelevant == -1) ? 0 : 1.0d / firstRelevant;
EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, reciprocalRank);
evalQueryQuality.setMetricDetails(new Breakdown(firstRelevant));
evalQueryQuality.addHitsAndRatings(ratedHits);
return evalQueryQuality;
}
private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold");
private static final ParseField K_FIELD = new ParseField("k");
private static final ConstructingObjectParser<MeanReciprocalRank, Void> PARSER = new ConstructingObjectParser<>("reciprocal_rank",
args -> {
Integer optionalThreshold = (Integer) args[0];
Integer optionalK = (Integer) args[1];
return new MeanReciprocalRank(optionalThreshold == null ? DEFAULT_RATING_THRESHOLD : optionalThreshold,
optionalK == null ? DEFAULT_K : optionalK);
});
static {
PARSER.declareInt(optionalConstructorArg(), RELEVANT_RATING_FIELD);
PARSER.declareInt(optionalConstructorArg(), K_FIELD);
}
public static MeanReciprocalRank fromXContent(XContentParser parser) {
return PARSER.apply(parser, null);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject(NAME);
builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold);
builder.field(K_FIELD.getPreferredName(), this.k);
builder.endObject();
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
MeanReciprocalRank other = (MeanReciprocalRank) obj;
return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold)
&& Objects.equals(k, other.k);
}
@Override
public final int hashCode() {
return Objects.hash(relevantRatingThreshhold, k);
}
static class Breakdown implements MetricDetails {
private final int firstRelevantRank;
Breakdown(int firstRelevantRank) {
this.firstRelevantRank = firstRelevantRank;
}
Breakdown(StreamInput in) throws IOException {
this.firstRelevantRank = in.readVInt();
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params)
throws IOException {
builder.field("first_relevant", firstRelevantRank);
return builder;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(firstRelevantRank);
}
@Override
public String getWriteableName() {
return NAME;
}
int getFirstRelevantRank() {
return firstRelevantRank;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
MeanReciprocalRank.Breakdown other = (MeanReciprocalRank.Breakdown) obj;
return Objects.equals(firstRelevantRank, other.firstRelevantRank);
}
@Override
public final int hashCode() {
return Objects.hash(firstRelevantRank);
}
}
}

30
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/MetricDetails.java Normal file
View File

@ -0,0 +1,30 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.xcontent.ToXContent;
/**
* Details about a specific {@link EvaluationMetric} that should be included in the resonse.
*/
public interface MetricDetails extends ToXContent, NamedWriteable {
}

281
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtK.java Normal file
View File

@ -0,0 +1,281 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchHit;
import java.io.IOException;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import javax.naming.directory.SearchResult;
import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg;
import static org.elasticsearch.index.rankeval.EvaluationMetric.joinHitsWithRatings;
/**
* Metric implementing Precision@K
* (https://en.wikipedia.org/wiki/Information_retrieval#Precision_at_K).<br>
* By default documents with a rating equal or bigger than 1 are considered to
* be "relevant" for this calculation. This value can be changes using the
* relevant_rating_threshold` parameter.<br>
* The `ignore_unlabeled` parameter (default to false) controls if unrated
* documents should be ignored.
* The `k` parameter (defaults to 10) controls the search window size.
*/
public class PrecisionAtK implements EvaluationMetric {
public static final String NAME = "precision";
private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold");
private static final ParseField IGNORE_UNLABELED_FIELD = new ParseField("ignore_unlabeled");
private static final ParseField K_FIELD = new ParseField("k");
private static final int DEFAULT_K = 10;
private final boolean ignoreUnlabeled;
private final int relevantRatingThreshhold;
private final int k;
/**
* Metric implementing Precision@K.
* @param threshold
* ratings equal or above this value will be considered relevant.
* @param ignoreUnlabeled
* Controls how unlabeled documents in the search hits are treated.
* Set to 'true', unlabeled documents are ignored and neither count
* as true or false positives. Set to 'false', they are treated as
* false positives.
* @param k
* controls the window size for the search results the metric takes into account
*/
public PrecisionAtK(int threshold, boolean ignoreUnlabeled, int k) {
if (threshold < 0) {
throw new IllegalArgumentException("Relevant rating threshold for precision must be positive integer.");
}
if (k <= 0) {
throw new IllegalArgumentException("Window size k must be positive.");
}
this.relevantRatingThreshhold = threshold;
this.ignoreUnlabeled = ignoreUnlabeled;
this.k = k;
}
public PrecisionAtK() {
this(1, false, DEFAULT_K);
}
private static final ConstructingObjectParser<PrecisionAtK, Void> PARSER = new ConstructingObjectParser<>(NAME,
args -> {
Integer threshHold = (Integer) args[0];
Boolean ignoreUnlabeled = (Boolean) args[1];
Integer k = (Integer) args[2];
return new PrecisionAtK(threshHold == null ? 1 : threshHold,
ignoreUnlabeled == null ? false : ignoreUnlabeled,
k == null ? DEFAULT_K : k);
});
static {
PARSER.declareInt(optionalConstructorArg(), RELEVANT_RATING_FIELD);
PARSER.declareBoolean(optionalConstructorArg(), IGNORE_UNLABELED_FIELD);
PARSER.declareInt(optionalConstructorArg(), K_FIELD);
}
PrecisionAtK(StreamInput in) throws IOException {
relevantRatingThreshhold = in.readVInt();
ignoreUnlabeled = in.readBoolean();
k = in.readVInt();
}
int getK() {
return this.k;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(relevantRatingThreshhold);
out.writeBoolean(ignoreUnlabeled);
out.writeVInt(k);
}
@Override
public String getWriteableName() {
return NAME;
}
/**
* Return the rating threshold above which ratings are considered to be
* "relevant" for this metric. Defaults to 1.
*/
public int getRelevantRatingThreshold() {
return relevantRatingThreshhold;
}
/**
* Gets the 'ignore_unlabeled' parameter.
*/
public boolean getIgnoreUnlabeled() {
return ignoreUnlabeled;
}
@Override
public Optional<Integer> forcedSearchSize() {
return Optional.of(k);
}
public static PrecisionAtK fromXContent(XContentParser parser) {
return PARSER.apply(parser, null);
}
/**
* Compute precisionAtN based on provided relevant document IDs.
*
* @return precision at n for above {@link SearchResult} list.
**/
@Override
public EvalQueryQuality evaluate(String taskId, SearchHit[] hits,
List<RatedDocument> ratedDocs) {
int truePositives = 0;
int falsePositives = 0;
List<RatedSearchHit> ratedSearchHits = joinHitsWithRatings(hits, ratedDocs);
for (RatedSearchHit hit : ratedSearchHits) {
Optional<Integer> rating = hit.getRating();
if (rating.isPresent()) {
if (rating.get() >= this.relevantRatingThreshhold) {
truePositives++;
} else {
falsePositives++;
}
} else if (ignoreUnlabeled == false) {
falsePositives++;
}
}
double precision = 0.0;
if (truePositives + falsePositives > 0) {
precision = (double) truePositives / (truePositives + falsePositives);
}
EvalQueryQuality evalQueryQuality = new EvalQueryQuality(taskId, precision);
evalQueryQuality.setMetricDetails(
new PrecisionAtK.Breakdown(truePositives, truePositives + falsePositives));
evalQueryQuality.addHitsAndRatings(ratedSearchHits);
return evalQueryQuality;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject(NAME);
builder.field(RELEVANT_RATING_FIELD.getPreferredName(), this.relevantRatingThreshhold);
builder.field(IGNORE_UNLABELED_FIELD.getPreferredName(), this.ignoreUnlabeled);
builder.field(K_FIELD.getPreferredName(), this.k);
builder.endObject();
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
PrecisionAtK other = (PrecisionAtK) obj;
return Objects.equals(relevantRatingThreshhold, other.relevantRatingThreshhold)
&& Objects.equals(k, other.k)
&& Objects.equals(ignoreUnlabeled, other.ignoreUnlabeled);
}
@Override
public final int hashCode() {
return Objects.hash(relevantRatingThreshhold, ignoreUnlabeled, k);
}
static class Breakdown implements MetricDetails {
private static final String DOCS_RETRIEVED_FIELD = "docs_retrieved";
private static final String RELEVANT_DOCS_RETRIEVED_FIELD = "relevant_docs_retrieved";
private int relevantRetrieved;
private int retrieved;
Breakdown(int relevantRetrieved, int retrieved) {
this.relevantRetrieved = relevantRetrieved;
this.retrieved = retrieved;
}
Breakdown(StreamInput in) throws IOException {
this.relevantRetrieved = in.readVInt();
this.retrieved = in.readVInt();
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params)
throws IOException {
builder.field(RELEVANT_DOCS_RETRIEVED_FIELD, relevantRetrieved);
builder.field(DOCS_RETRIEVED_FIELD, retrieved);
return builder;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(relevantRetrieved);
out.writeVInt(retrieved);
}
@Override
public String getWriteableName() {
return NAME;
}
int getRelevantRetrieved() {
return relevantRetrieved;
}
int getRetrieved() {
return retrieved;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
PrecisionAtK.Breakdown other = (PrecisionAtK.Breakdown) obj;
return Objects.equals(relevantRetrieved, other.relevantRetrieved)
&& Objects.equals(retrieved, other.retrieved);
}
@Override
public final int hashCode() {
return Objects.hash(relevantRetrieved, retrieved);
}
}
}

46
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalAction.java Normal file
View File

@ -0,0 +1,46 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.Action;
import org.elasticsearch.client.ElasticsearchClient;
/**
* Action for explaining evaluating search ranking results.
*/
public class RankEvalAction extends Action<RankEvalRequest, RankEvalResponse, RankEvalRequestBuilder> {
public static final RankEvalAction INSTANCE = new RankEvalAction();
public static final String NAME = "indices:data/read/rank_eval";
private RankEvalAction() {
super(NAME);
}
@Override
public RankEvalRequestBuilder newRequestBuilder(ElasticsearchClient client) {
return new RankEvalRequestBuilder(client, this, new RankEvalRequest());
}
@Override
public RankEvalResponse newResponse() {
return new RankEvalResponse();
}
}

42
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalNamedXContentProvider.java Normal file
View File

@ -0,0 +1,42 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.plugins.spi.NamedXContentProvider;
import java.util.ArrayList;
import java.util.List;
public class RankEvalNamedXContentProvider implements NamedXContentProvider {
@Override
public List<NamedXContentRegistry.Entry> getNamedXContentParsers() {
List<NamedXContentRegistry.Entry> namedXContent = new ArrayList<>();
namedXContent.add(
new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(PrecisionAtK.NAME), PrecisionAtK::fromXContent));
namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(MeanReciprocalRank.NAME),
MeanReciprocalRank::fromXContent));
namedXContent.add(new NamedXContentRegistry.Entry(EvaluationMetric.class, new ParseField(DiscountedCumulativeGain.NAME),
DiscountedCumulativeGain::fromXContent));
return namedXContent;
}
}

73
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalPlugin.java Normal file
View File

@ -0,0 +1,73 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.cluster.node.DiscoveryNodes;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.ClusterSettings;
import org.elasticsearch.common.settings.IndexScopedSettings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.settings.SettingsFilter;
import org.elasticsearch.common.xcontent.NamedXContentRegistry.Entry;
import org.elasticsearch.plugins.ActionPlugin;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestHandler;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.function.Supplier;
public class RankEvalPlugin extends Plugin implements ActionPlugin {
@Override
public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
return Arrays.asList(new ActionHandler<>(RankEvalAction.INSTANCE, TransportRankEvalAction.class));
}
@Override
public List<RestHandler> getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings,
IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<DiscoveryNodes> nodesInCluster) {
return Arrays.asList(new RestRankEvalAction(settings, restController));
}
@Override
public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, PrecisionAtK.NAME, PrecisionAtK::new));
namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new));
namedWriteables.add(
new NamedWriteableRegistry.Entry(EvaluationMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new));
namedWriteables.add(new NamedWriteableRegistry.Entry(MetricDetails.class, PrecisionAtK.NAME, PrecisionAtK.Breakdown::new));
namedWriteables
.add(new NamedWriteableRegistry.Entry(MetricDetails.class, MeanReciprocalRank.NAME, MeanReciprocalRank.Breakdown::new));
return namedWriteables;
}
@Override
public List<Entry> getNamedXContent() {
return new RankEvalNamedXContentProvider().getNamedXContentParsers();
}
}

72
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java Normal file
View File

@ -0,0 +1,72 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.ActionRequest;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import java.io.IOException;
/**
* Request to perform a search ranking evaluation.
*/
public class RankEvalRequest extends ActionRequest {
private RankEvalSpec rankingEvaluation;
@Override
public ActionRequestValidationException validate() {
ActionRequestValidationException e = null;
if (rankingEvaluation == null) {
e = new ActionRequestValidationException();
e.addValidationError("missing ranking evaluation specification");
}
return e;
}
/**
* Returns the specification of the ranking evaluation.
*/
public RankEvalSpec getRankEvalSpec() {
return rankingEvaluation;
}
/**
* Set the the specification of the ranking evaluation.
*/
public void setRankEvalSpec(RankEvalSpec task) {
this.rankingEvaluation = task;
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
rankingEvaluation = new RankEvalSpec(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
rankingEvaluation.writeTo(out);
}
}

45
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequestBuilder.java Normal file
View File

@ -0,0 +1,45 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.Action;
import org.elasticsearch.action.ActionRequestBuilder;
import org.elasticsearch.client.ElasticsearchClient;
public class RankEvalRequestBuilder extends ActionRequestBuilder<RankEvalRequest, RankEvalResponse, RankEvalRequestBuilder> {
public RankEvalRequestBuilder(ElasticsearchClient client, Action<RankEvalRequest, RankEvalResponse, RankEvalRequestBuilder> action,
RankEvalRequest request) {
super(client, action, request);
}
@Override
public RankEvalRequest request() {
return request;
}
public void setRankEvalSpec(RankEvalSpec spec) {
this.request.setRankEvalSpec(spec);
}
public RankEvalSpec getRankEvalSpec() {
return this.request.getRankEvalSpec();
}
}

133
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java Normal file
View File

@ -0,0 +1,133 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
/**
* Returns the results for a {@link RankEvalRequest}.<br>
* The repsonse contains a detailed section for each evaluation query in the request and
* possible failures that happened when executin individual queries.
**/
public class RankEvalResponse extends ActionResponse implements ToXContentObject {
/** The overall evaluation result. */
private double evaluationResult;
/** details about individual ranking evaluation queries, keyed by their id */
private Map<String, EvalQueryQuality> details;
/** exceptions for specific ranking evaluation queries, keyed by their id */
private Map<String, Exception> failures;
public RankEvalResponse(double qualityLevel, Map<String, EvalQueryQuality> partialResults,
Map<String, Exception> failures) {
this.evaluationResult = qualityLevel;
this.details = new HashMap<>(partialResults);
this.failures = new HashMap<>(failures);
}
RankEvalResponse() {
// only used in RankEvalAction#newResponse()
}
public double getEvaluationResult() {
return evaluationResult;
}
public Map<String, EvalQueryQuality> getPartialResults() {
return Collections.unmodifiableMap(details);
}
public Map<String, Exception> getFailures() {
return Collections.unmodifiableMap(failures);
}
@Override
public String toString() {
return Strings.toString(this);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeDouble(evaluationResult);
out.writeVInt(details.size());
for (String queryId : details.keySet()) {
out.writeString(queryId);
details.get(queryId).writeTo(out);
}
out.writeVInt(failures.size());
for (String queryId : failures.keySet()) {
out.writeString(queryId);
out.writeException(failures.get(queryId));
}
}
@Override
public void readFrom(StreamInput in) throws IOException {
super.readFrom(in);
this.evaluationResult = in.readDouble();
int partialResultSize = in.readVInt();
this.details = new HashMap<>(partialResultSize);
for (int i = 0; i < partialResultSize; i++) {
String queryId = in.readString();
EvalQueryQuality partial = new EvalQueryQuality(in);
this.details.put(queryId, partial);
}
int failuresSize = in.readVInt();
this.failures = new HashMap<>(failuresSize);
for (int i = 0; i < failuresSize; i++) {
String queryId = in.readString();
this.failures.put(queryId, in.readException());
}
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startObject("rank_eval");
builder.field("quality_level", evaluationResult);
builder.startObject("details");
for (String key : details.keySet()) {
details.get(key).toXContent(builder, params);
}
builder.endObject();
builder.startObject("failures");
for (String key : failures.keySet()) {
builder.startObject(key);
ElasticsearchException.generateFailureXContent(builder, params, failures.get(key), false);
builder.endObject();
}
builder.endObject();
builder.endObject();
builder.endObject();
return builder;
}
}

273
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java Normal file
View File

@ -0,0 +1,273 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParserUtils;
import org.elasticsearch.script.Script;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
/**
* Specification of the ranking evaluation request.<br>
* This class groups the queries to evaluate, including their document ratings,
* and the evaluation metric including its parameters.
*/
public class RankEvalSpec implements Writeable, ToXContentObject {
/** List of search request to use for the evaluation */
private final List<RatedRequest> ratedRequests;
/** Definition of the quality metric, e.g. precision at N */
private final EvaluationMetric metric;
/** Maximum number of requests to execute in parallel. */
private int maxConcurrentSearches = MAX_CONCURRENT_SEARCHES;
/** Default max number of requests. */
private static final int MAX_CONCURRENT_SEARCHES = 10;
/** optional: Templates to base test requests on */
private Map<String, Script> templates = new HashMap<>();
/** the indices this ranking evaluation targets */
private final List<String> indices;
public RankEvalSpec(List<RatedRequest> ratedRequests, EvaluationMetric metric, Collection<ScriptWithId> templates) {
this.metric = Objects.requireNonNull(metric, "Cannot evaluate ranking if no evaluation metric is provided.");
if (ratedRequests == null || ratedRequests.isEmpty()) {
throw new IllegalArgumentException(
"Cannot evaluate ranking if no search requests with rated results are provided. Seen: " + ratedRequests);
}
this.ratedRequests = ratedRequests;
if (templates == null || templates.isEmpty()) {
for (RatedRequest request : ratedRequests) {
if (request.getTestRequest() == null) {
throw new IllegalStateException("Cannot evaluate ranking if neither template nor test request is "
+ "provided. Seen for request id: " + request.getId());
}
}
}
if (templates != null) {
for (ScriptWithId idScript : templates) {
this.templates.put(idScript.id, idScript.script);
}
}
this.indices = new ArrayList<>();
}
public RankEvalSpec(List<RatedRequest> ratedRequests, EvaluationMetric metric) {
this(ratedRequests, metric, null);
}
public RankEvalSpec(StreamInput in) throws IOException {
int specSize = in.readVInt();
ratedRequests = new ArrayList<>(specSize);
for (int i = 0; i < specSize; i++) {
ratedRequests.add(new RatedRequest(in));
}
metric = in.readNamedWriteable(EvaluationMetric.class);
int size = in.readVInt();
for (int i = 0; i < size; i++) {
String key = in.readString();
Script value = new Script(in);
this.templates.put(key, value);
}
maxConcurrentSearches = in.readVInt();
int indicesSize = in.readInt();
indices = new ArrayList<>(indicesSize);
for (int i = 0; i < indicesSize; i++) {
this.indices.add(in.readString());
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeVInt(ratedRequests.size());
for (RatedRequest spec : ratedRequests) {
spec.writeTo(out);
}
out.writeNamedWriteable(metric);
out.writeVInt(templates.size());
for (Entry<String, Script> entry : templates.entrySet()) {
out.writeString(entry.getKey());
entry.getValue().writeTo(out);
}
out.writeVInt(maxConcurrentSearches);
out.writeInt(indices.size());
for (String index : indices) {
out.writeString(index);
}
}
/** Returns the metric to use for quality evaluation.*/
public EvaluationMetric getMetric() {
return metric;
}
/** Returns a list of intent to query translation specifications to evaluate. */
public List<RatedRequest> getRatedRequests() {
return Collections.unmodifiableList(ratedRequests);
}
/** Returns the template to base test requests on. */
public Map<String, Script> getTemplates() {
return this.templates;
}
/** Returns the max concurrent searches allowed. */
public int getMaxConcurrentSearches() {
return this.maxConcurrentSearches;
}
/** Set the max concurrent searches allowed. */
public void setMaxConcurrentSearches(int maxConcurrentSearches) {
this.maxConcurrentSearches = maxConcurrentSearches;
}
public void addIndices(List<String> indices) {
this.indices.addAll(indices);
}
public List<String> getIndices() {
return Collections.unmodifiableList(indices);
}
private static final ParseField TEMPLATES_FIELD = new ParseField("templates");
private static final ParseField METRIC_FIELD = new ParseField("metric");
private static final ParseField REQUESTS_FIELD = new ParseField("requests");
private static final ParseField MAX_CONCURRENT_SEARCHES_FIELD = new ParseField("max_concurrent_searches");
@SuppressWarnings("unchecked")
private static final ConstructingObjectParser<RankEvalSpec, Void> PARSER = new ConstructingObjectParser<>("rank_eval",
a -> new RankEvalSpec((List<RatedRequest>) a[0], (EvaluationMetric) a[1], (Collection<ScriptWithId>) a[2]));
static {
PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> RatedRequest.fromXContent(p), REQUESTS_FIELD);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> parseMetric(p), METRIC_FIELD);
PARSER.declareObjectArray(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> ScriptWithId.fromXContent(p),
TEMPLATES_FIELD);
PARSER.declareInt(RankEvalSpec::setMaxConcurrentSearches, MAX_CONCURRENT_SEARCHES_FIELD);
}
private static EvaluationMetric parseMetric(XContentParser parser) throws IOException {
XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
XContentParserUtils.ensureExpectedToken(XContentParser.Token.FIELD_NAME, parser.nextToken(), parser::getTokenLocation);
EvaluationMetric metric = parser.namedObject(EvaluationMetric.class, parser.currentName(), null);
XContentParserUtils.ensureExpectedToken(XContentParser.Token.END_OBJECT, parser.nextToken(), parser::getTokenLocation);
return metric;
}
public static RankEvalSpec parse(XContentParser parser) {
return PARSER.apply(parser, null);
}
static class ScriptWithId {
private Script script;
private String id;
private static final ParseField TEMPLATE_FIELD = new ParseField("template");
private static final ParseField TEMPLATE_ID_FIELD = new ParseField("id");
ScriptWithId(String id, Script script) {
this.id = id;
this.script = script;
}
private static final ConstructingObjectParser<ScriptWithId, Void> PARSER =
new ConstructingObjectParser<>("script_with_id",
a -> new ScriptWithId((String) a[0], (Script) a[1]));
public static ScriptWithId fromXContent(XContentParser parser) {
return PARSER.apply(parser, null);
}
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), TEMPLATE_ID_FIELD);
PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> {
try {
return Script.parse(p, "mustache");
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex);
}
}, TEMPLATE_FIELD);
}
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.startArray(TEMPLATES_FIELD.getPreferredName());
for (Entry<String, Script> entry : templates.entrySet()) {
builder.startObject();
builder.field(ScriptWithId.TEMPLATE_ID_FIELD.getPreferredName(), entry.getKey());
builder.field(ScriptWithId.TEMPLATE_FIELD.getPreferredName(), entry.getValue());
builder.endObject();
}
builder.endArray();
builder.startArray(REQUESTS_FIELD.getPreferredName());
for (RatedRequest spec : this.ratedRequests) {
spec.toXContent(builder, params);
}
builder.endArray();
builder.field(METRIC_FIELD.getPreferredName(), this.metric);
builder.field(MAX_CONCURRENT_SEARCHES_FIELD.getPreferredName(), maxConcurrentSearches);
builder.endObject();
return builder;
}
@Override
public String toString() {
return Strings.toString(this);
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
RankEvalSpec other = (RankEvalSpec) obj;
return Objects.equals(ratedRequests, other.ratedRequests) &&
Objects.equals(metric, other.metric) &&
Objects.equals(maxConcurrentSearches, other.maxConcurrentSearches) &&
Objects.equals(templates, other.templates) &&
Objects.equals(indices, other.indices);
}
@Override
public final int hashCode() {
return Objects.hash(ratedRequests, metric, templates, maxConcurrentSearches, indices);
}
}

186
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java Normal file
View File

@ -0,0 +1,186 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.Objects;
/**
* Represents a document (specified by its _index/_id) and its corresponding rating
* with respect to a specific search query.
* <p>
* The json structure of this element in a request:
* <pre>
* {
* "_index": "my_index",
* "_id": "doc1",
* "rating": 0
* }
* </pre>
*
*/
public class RatedDocument implements Writeable, ToXContentObject {
static final ParseField RATING_FIELD = new ParseField("rating");
static final ParseField DOC_ID_FIELD = new ParseField("_id");
static final ParseField INDEX_FIELD = new ParseField("_index");
private static final ConstructingObjectParser<RatedDocument, Void> PARSER = new ConstructingObjectParser<>("rated_document",
a -> new RatedDocument((String) a[0], (String) a[1], (Integer) a[2]));
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD);
PARSER.declareString(ConstructingObjectParser.constructorArg(), DOC_ID_FIELD);
PARSER.declareInt(ConstructingObjectParser.constructorArg(), RATING_FIELD);
}
private final int rating;
private final DocumentKey key;
public RatedDocument(String index, String id, int rating) {
this.key = new DocumentKey(index, id);
this.rating = rating;
}
RatedDocument(StreamInput in) throws IOException {
this.key = new DocumentKey(in.readString(), in.readString());
this.rating = in.readVInt();
}
public DocumentKey getKey() {
return this.key;
}
public String getIndex() {
return key.getIndex();
}
public String getDocID() {
return key.getDocId();
}
public int getRating() {
return rating;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(key.getIndex());
out.writeString(key.getDocId());
out.writeVInt(rating);
}
static RatedDocument fromXContent(XContentParser parser) {
return PARSER.apply(parser, null);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(INDEX_FIELD.getPreferredName(), key.getIndex());
builder.field(DOC_ID_FIELD.getPreferredName(), key.getDocId());
builder.field(RATING_FIELD.getPreferredName(), rating);
builder.endObject();
return builder;
}
@Override
public String toString() {
return Strings.toString(this);
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
RatedDocument other = (RatedDocument) obj;
return Objects.equals(key, other.key) && Objects.equals(rating, other.rating);
}
@Override
public final int hashCode() {
return Objects.hash(key, rating);
}
/**
* a joint document key consisting of the documents index and id
*/
static class DocumentKey {
private final String docId;
private final String index;
DocumentKey(String index, String docId) {
if (Strings.isNullOrEmpty(index)) {
throw new IllegalArgumentException("Index must be set for each rated document");
}
if (Strings.isNullOrEmpty(docId)) {
throw new IllegalArgumentException("DocId must be set for each rated document");
}
this.index = index;
this.docId = docId;
}
String getIndex() {
return index;
}
String getDocId() {
return docId;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
DocumentKey other = (DocumentKey) obj;
return Objects.equals(index, other.index) && Objects.equals(docId, other.docId);
}
@Override
public final int hashCode() {
return Objects.hash(index, docId);
}
@Override
public String toString() {
return "{\"_index\":\"" + index + "\",\"_id\":\"" + docId + "\"}";
}
}
}

300
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java Normal file
View File

@ -0,0 +1,300 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.Nullable;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
import java.util.Set;
/**
* Definition of a particular query in the ranking evaluation request.<br>
* This usually represents a single user search intent and consists of an id
* (ideally human readable and referencing the search intent), the list of
* indices to be queries and the {@link SearchSourceBuilder} that will be used
* to create the search request for this search intent.<br>
* Alternatively, a template id and template parameters can be provided instead.<br>
* Finally, a list of rated documents for this query also needs to be provided.
* <p>
* The json structure in the rest request looks like this:
* <pre>
* {
* "id": "coffee_query",
* "request": {
* "query": {
* "match": { "beverage": "coffee" }
* }
* },
* "summary_fields": ["title"],
* "ratings": [
* {"_index": "my_index", "_id": "doc1", "rating": 0},
* {"_index": "my_index", "_id": "doc2","rating": 3},
* {"_index": "my_index", "_id": "doc3", "rating": 1}
* ]
* }
* </pre>
*/
@SuppressWarnings("unchecked")
public class RatedRequest implements Writeable, ToXContentObject {
private final String id;
private final List<String> summaryFields;
private final List<RatedDocument> ratedDocs;
// Search request to execute for this rated request. This can be null if template and corresponding parameters are supplied.
@Nullable
private SearchSourceBuilder testRequest;
/**
* Map of parameters to use for filling a query template, can be used
* instead of providing testRequest.
*/
private final Map<String, Object> params;
@Nullable
private String templateId;
private RatedRequest(String id, List<RatedDocument> ratedDocs, SearchSourceBuilder testRequest,
Map<String, Object> params, String templateId) {
if (params != null && (params.size() > 0 && testRequest != null)) {
throw new IllegalArgumentException(
"Ambiguous rated request: Set both, verbatim test request and test request "
+ "template parameters.");
}
if (templateId != null && testRequest != null) {
throw new IllegalArgumentException(
"Ambiguous rated request: Set both, verbatim test request and test request "
+ "template parameters.");
}
if ((params == null || params.size() < 1) && testRequest == null) {
throw new IllegalArgumentException(
"Need to set at least test request or test request template parameters.");
}
if ((params != null && params.size() > 0) && templateId == null) {
throw new IllegalArgumentException(
"If template parameters are supplied need to set id of template to apply "
+ "them to too.");
}
// check that not two documents with same _index/id are specified
Set<DocumentKey> docKeys = new HashSet<>();
for (RatedDocument doc : ratedDocs) {
if (docKeys.add(doc.getKey()) == false) {
String docKeyToString = doc.getKey().toString().replaceAll("\n", "").replaceAll(" ", " ");
throw new IllegalArgumentException(
"Found duplicate rated document key [" + docKeyToString + "] in evaluation request [" + id + "]");
}
}
this.id = id;
this.testRequest = testRequest;
this.ratedDocs = new ArrayList<>(ratedDocs);
if (params != null) {
this.params = new HashMap<>(params);
} else {
this.params = Collections.emptyMap();
}
this.templateId = templateId;
this.summaryFields = new ArrayList<>();
}
public RatedRequest(String id, List<RatedDocument> ratedDocs, Map<String, Object> params,
String templateId) {
this(id, ratedDocs, null, params, templateId);
}
public RatedRequest(String id, List<RatedDocument> ratedDocs, SearchSourceBuilder testRequest) {
this(id, ratedDocs, testRequest, new HashMap<>(), null);
}
public RatedRequest(StreamInput in) throws IOException {
this.id = in.readString();
testRequest = in.readOptionalWriteable(SearchSourceBuilder::new);
int intentSize = in.readInt();
ratedDocs = new ArrayList<>(intentSize);
for (int i = 0; i < intentSize; i++) {
ratedDocs.add(new RatedDocument(in));
}
this.params = in.readMap();
int summaryFieldsSize = in.readInt();
summaryFields = new ArrayList<>(summaryFieldsSize);
for (int i = 0; i < summaryFieldsSize; i++) {
this.summaryFields.add(in.readString());
}
this.templateId = in.readOptionalString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(id);
out.writeOptionalWriteable(testRequest);
out.writeInt(ratedDocs.size());
for (RatedDocument ratedDoc : ratedDocs) {
ratedDoc.writeTo(out);
}
out.writeMap(params);
out.writeInt(summaryFields.size());
for (String fieldName : summaryFields) {
out.writeString(fieldName);
}
out.writeOptionalString(this.templateId);
}
public SearchSourceBuilder getTestRequest() {
return testRequest;
}
/** return the user supplied request id */
public String getId() {
return id;
}
/** return the list of rated documents to evaluate. */
public List<RatedDocument> getRatedDocs() {
return Collections.unmodifiableList(ratedDocs);
}
/** return the parameters if this request uses a template, otherwise this will be empty. */
public Map<String, Object> getParams() {
return Collections.unmodifiableMap(this.params);
}
/** return the parameters if this request uses a template, otherwise this will be <tt>null</tt>. */
public String getTemplateId() {
return this.templateId;
}
/** returns a list of fields that should be included in the document summary for matched documents */
public List<String> getSummaryFields() {
return Collections.unmodifiableList(summaryFields);
}
public void addSummaryFields(List<String> summaryFields) {
this.summaryFields.addAll(Objects.requireNonNull(summaryFields, "no summary fields supplied"));
}
private static final ParseField ID_FIELD = new ParseField("id");
private static final ParseField REQUEST_FIELD = new ParseField("request");
private static final ParseField RATINGS_FIELD = new ParseField("ratings");
private static final ParseField PARAMS_FIELD = new ParseField("params");
private static final ParseField FIELDS_FIELD = new ParseField("summary_fields");
private static final ParseField TEMPLATE_ID_FIELD = new ParseField("template_id");
private static final ConstructingObjectParser<RatedRequest, Void> PARSER = new ConstructingObjectParser<>("requests",
a -> new RatedRequest((String) a[0], (List<RatedDocument>) a[1], (SearchSourceBuilder) a[2], (Map<String, Object>) a[3],
(String) a[4]));
static {
PARSER.declareString(ConstructingObjectParser.constructorArg(), ID_FIELD);
PARSER.declareObjectArray(ConstructingObjectParser.constructorArg(), (p, c) -> {
return RatedDocument.fromXContent(p);
}, RATINGS_FIELD);
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) ->
SearchSourceBuilder.fromXContent(p), REQUEST_FIELD);
PARSER.declareObject(ConstructingObjectParser.optionalConstructorArg(), (p, c) -> p.map(), PARAMS_FIELD);
PARSER.declareStringArray(RatedRequest::addSummaryFields, FIELDS_FIELD);
PARSER.declareString(ConstructingObjectParser.optionalConstructorArg(), TEMPLATE_ID_FIELD);
}
/**
* parse from rest representation
*/
public static RatedRequest fromXContent(XContentParser parser) {
return PARSER.apply(parser, null);
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
builder.field(ID_FIELD.getPreferredName(), this.id);
if (testRequest != null) {
builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest);
}
builder.startArray(RATINGS_FIELD.getPreferredName());
for (RatedDocument doc : this.ratedDocs) {
doc.toXContent(builder, params);
}
builder.endArray();
if (this.templateId != null) {
builder.field(TEMPLATE_ID_FIELD.getPreferredName(), this.templateId);
}
if (this.params.isEmpty() == false) {
builder.startObject(PARAMS_FIELD.getPreferredName());
for (Entry<String, Object> entry : this.params.entrySet()) {
builder.field(entry.getKey(), entry.getValue());
}
builder.endObject();
}
if (this.summaryFields.isEmpty() == false) {
builder.startArray(FIELDS_FIELD.getPreferredName());
for (String field : this.summaryFields) {
builder.value(field);
}
builder.endArray();
}
builder.endObject();
return builder;
}
@Override
public String toString() {
return Strings.toString(this);
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
RatedRequest other = (RatedRequest) obj;
return Objects.equals(id, other.id) && Objects.equals(testRequest, other.testRequest)
&& Objects.equals(summaryFields, other.summaryFields)
&& Objects.equals(ratedDocs, other.ratedDocs)
&& Objects.equals(params, other.params)
&& Objects.equals(templateId, other.templateId);
}
@Override
public final int hashCode() {
return Objects.hash(id, testRequest, summaryFields, ratedDocs, params,
templateId);
}
}

95
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedSearchHit.java Normal file
View File

@ -0,0 +1,95 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.search.SearchHit;
import java.io.IOException;
import java.util.Objects;
import java.util.Optional;
/**
* Combines a {@link SearchHit} with a document rating.
*/
public class RatedSearchHit implements Writeable, ToXContent {
private final SearchHit searchHit;
private final Optional<Integer> rating;
public RatedSearchHit(SearchHit searchHit, Optional<Integer> rating) {
this.searchHit = searchHit;
this.rating = rating;
}
RatedSearchHit(StreamInput in) throws IOException {
this(SearchHit.readSearchHit(in),
in.readBoolean() == true ? Optional.of(in.readVInt()) : Optional.empty());
}
@Override
public void writeTo(StreamOutput out) throws IOException {
searchHit.writeTo(out);
out.writeBoolean(rating.isPresent());
if (rating.isPresent()) {
out.writeVInt(rating.get());
}
}
public SearchHit getSearchHit() {
return this.searchHit;
}
public Optional<Integer> getRating() {
return this.rating;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params params)
throws IOException {
builder.startObject();
builder.field("hit", (ToXContent) searchHit);
builder.field("rating", rating.orElse(null));
builder.endObject();
return builder;
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
RatedSearchHit other = (RatedSearchHit) obj;
return Objects.equals(rating, other.rating)
&& Objects.equals(searchHit, other.searchHit);
}
@Override
public final int hashCode() {
return Objects.hash(rating, searchHit);
}
}

121
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java Normal file
View File

@ -0,0 +1,121 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.action.RestToXContentListener;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import static org.elasticsearch.rest.RestRequest.Method.GET;
import static org.elasticsearch.rest.RestRequest.Method.POST;
/**
* {
* "requests": [{
* "id": "amsterdam_query",
* "request": {
* "query": {
* "match": {
* "text": "amsterdam"
* }
* }
* },
* "ratings": [{
* "_index": "foo",
* "_id": "doc1",
* "rating": 0
* },
* {
* "_index": "foo",
* "_id": "doc2",
* "rating": 1
* },
* {
* "_index": "foo",
* "_id": "doc3",
* "rating": 1
* }
* ]
* },
* {
* "id": "berlin_query",
* "request": {
* "query": {
* "match": {
* "text": "berlin"
* }
* },
* "size": 10
* },
* "ratings": [{
* "_index": "foo",
* "_id": "doc1",
* "rating": 1
* }]
* }
* ],
* "metric": {
* "precision": {
* "ignore_unlabeled": true
* }
* }
* }
*/
public class RestRankEvalAction extends BaseRestHandler {
public RestRankEvalAction(Settings settings, RestController controller) {
super(settings);
controller.registerHandler(GET, "/_rank_eval", this);
controller.registerHandler(POST, "/_rank_eval", this);
controller.registerHandler(GET, "/{index}/_rank_eval", this);
controller.registerHandler(POST, "/{index}/_rank_eval", this);
}
@Override
protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException {
RankEvalRequest rankEvalRequest = new RankEvalRequest();
try (XContentParser parser = request.contentOrSourceParamParser()) {
parseRankEvalRequest(rankEvalRequest, request, parser);
}
return channel -> client.executeLocally(RankEvalAction.INSTANCE, rankEvalRequest,
new RestToXContentListener<RankEvalResponse>(channel));
}
private static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, XContentParser parser) {
List<String> indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
RankEvalSpec spec = RankEvalSpec.parse(parser);
spec.addIndices(indices);
rankEvalRequest.setRankEvalSpec(spec);
}
@Override
public String getName() {
return "rank_eval_action";
}
}

174
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java Normal file
View File

@ -0,0 +1,174 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.ActionListener;
import org.elasticsearch.action.search.MultiSearchRequest;
import org.elasticsearch.action.search.MultiSearchResponse;
import org.elasticsearch.action.search.MultiSearchResponse.Item;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.support.ActionFilters;
import org.elasticsearch.action.support.HandledTransportAction;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
import org.elasticsearch.common.bytes.BytesArray;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptService;
import org.elasticsearch.script.TemplateScript;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.threadpool.ThreadPool;
import org.elasticsearch.transport.TransportService;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
import static org.elasticsearch.common.xcontent.XContentHelper.createParser;
/**
* Instances of this class execute a collection of search intents (read: user
* supplied query parameters) against a set of possible search requests (read:
* search specifications, expressed as query/search request templates) and
* compares the result against a set of annotated documents per search intent.
*
* If any documents are returned that haven't been annotated the document id of
* those is returned per search intent.
*
* The resulting search quality is computed in terms of precision at n and
* returned for each search specification for the full set of search intents as
* averaged precision at n.
*/
public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequest, RankEvalResponse> {
private Client client;
private ScriptService scriptService;
private NamedXContentRegistry namedXContentRegistry;
@Inject
public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
IndexNameExpressionResolver indexNameExpressionResolver, Client client, TransportService transportService,
ScriptService scriptService, NamedXContentRegistry namedXContentRegistry) {
super(settings, RankEvalAction.NAME, threadPool, transportService, actionFilters, indexNameExpressionResolver,
RankEvalRequest::new);
this.scriptService = scriptService;
this.namedXContentRegistry = namedXContentRegistry;
this.client = client;
}
@Override
protected void doExecute(RankEvalRequest request, ActionListener<RankEvalResponse> listener) {
RankEvalSpec evaluationSpecification = request.getRankEvalSpec();
List<String> indices = evaluationSpecification.getIndices();
EvaluationMetric metric = evaluationSpecification.getMetric();
List<RatedRequest> ratedRequests = evaluationSpecification.getRatedRequests();
Map<String, Exception> errors = new ConcurrentHashMap<>(ratedRequests.size());
Map<String, TemplateScript.Factory> scriptsWithoutParams = new HashMap<>();
for (Entry<String, Script> entry : evaluationSpecification.getTemplates().entrySet()) {
scriptsWithoutParams.put(entry.getKey(), scriptService.compile(entry.getValue(), TemplateScript.CONTEXT));
}
MultiSearchRequest msearchRequest = new MultiSearchRequest();
msearchRequest.maxConcurrentSearchRequests(evaluationSpecification.getMaxConcurrentSearches());
List<RatedRequest> ratedRequestsInSearch = new ArrayList<>();
for (RatedRequest ratedRequest : ratedRequests) {
SearchSourceBuilder ratedSearchSource = ratedRequest.getTestRequest();
if (ratedSearchSource == null) {
Map<String, Object> params = ratedRequest.getParams();
String templateId = ratedRequest.getTemplateId();
TemplateScript.Factory templateScript = scriptsWithoutParams.get(templateId);
String resolvedRequest = templateScript.newInstance(params).execute();
try (XContentParser subParser = createParser(namedXContentRegistry, new BytesArray(resolvedRequest), XContentType.JSON)) {
ratedSearchSource = SearchSourceBuilder.fromXContent(subParser);
} catch (IOException e) {
// if we fail parsing, put the exception into the errors map and continue
errors.put(ratedRequest.getId(), e);
continue;
}
}
if (metric.forcedSearchSize().isPresent()) {
ratedSearchSource.size(metric.forcedSearchSize().get());
}
ratedRequestsInSearch.add(ratedRequest);
List<String> summaryFields = ratedRequest.getSummaryFields();
if (summaryFields.isEmpty()) {
ratedSearchSource.fetchSource(false);
} else {
ratedSearchSource.fetchSource(summaryFields.toArray(new String[summaryFields.size()]), new String[0]);
}
msearchRequest.add(new SearchRequest(indices.toArray(new String[indices.size()]), ratedSearchSource));
}
assert ratedRequestsInSearch.size() == msearchRequest.requests().size();
client.multiSearch(msearchRequest, new RankEvalActionListener(listener, metric,
ratedRequestsInSearch.toArray(new RatedRequest[ratedRequestsInSearch.size()]), errors));
}
class RankEvalActionListener implements ActionListener<MultiSearchResponse> {
private final ActionListener<RankEvalResponse> listener;
private final RatedRequest[] specifications;
private final Map<String, Exception> errors;
private final EvaluationMetric metric;
RankEvalActionListener(ActionListener<RankEvalResponse> listener, EvaluationMetric metric, RatedRequest[] specifications,
Map<String, Exception> errors) {
this.listener = listener;
this.metric = metric;
this.errors = errors;
this.specifications = specifications;
}
@Override
public void onResponse(MultiSearchResponse multiSearchResponse) {
int responsePosition = 0;
Map<String, EvalQueryQuality> responseDetails = new HashMap<>(specifications.length);
for (Item response : multiSearchResponse.getResponses()) {
RatedRequest specification = specifications[responsePosition];
if (response.isFailure() == false) {
SearchHit[] hits = response.getResponse().getHits().getHits();
EvalQueryQuality queryQuality = this.metric.evaluate(specification.getId(), hits, specification.getRatedDocs());
responseDetails.put(specification.getId(), queryQuality);
} else {
errors.put(specification.getId(), response.getFailure());
}
responsePosition++;
}
listener.onResponse(new RankEvalResponse(this.metric.combine(responseDetails.values()), responseDetails, this.errors));
}
@Override
public void onFailure(Exception exception) {
listener.onFailure(exception);
}
}
}

1
modules/rank-eval/src/main/resources/META-INF/services/org.elasticsearch.plugins.spi.NamedXContentProvider Normal file
View File

@ -0,0 +1 @@
org.elasticsearch.index.rankeval.RankEvalNamedXContentProvider

275
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainTests.java Normal file
View File

@ -0,0 +1,275 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.Index;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class DiscountedCumulativeGainTests extends ESTestCase {
static final double EXPECTED_DCG = 13.84826362927298;
static final double EXPECTED_IDCG = 14.595390756454922;
static final double EXPECTED_NDCG = EXPECTED_DCG / EXPECTED_IDCG;
private static final double DELTA = 10E-16;
/**
* Assuming the docs are ranked in the following order:
*
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1)
* -------------------------------------------------------------------------------------------
* 1 | 3 | 7.0 | 1.0 | 7.0 | 7.0 | 
* 2 | 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
* 3 | 3 | 7.0 | 2.0 | 3.5
* 4 | 0 | 0.0 | 2.321928094887362 | 0.0
* 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163
* 6 | 2 | 3.0 | 2.807354922057604 | 1.0686215613240666
*
* dcg = 13.84826362927298 (sum of last column)
*/
public void testDCGAt() {
List<RatedDocument> rated = new ArrayList<>();
int[] relevanceRatings = new int[] { 3, 2, 3, 0, 1, 2 };
SearchHit[] hits = new SearchHit[6];
for (int i = 0; i < 6; i++) {
rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
}
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
assertEquals(EXPECTED_DCG, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
/**
* Check with normalization: to get the maximal possible dcg, sort documents by
* relevance in descending order
*
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1)
* ---------------------------------------------------------------------------------------
* 1 | 3 | 7.0 | 1.0  | 7.0
* 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
* 3 | 2 | 3.0 | 2.0  | 1.5
* 4 | 2 | 3.0 | 2.321928094887362 | 1.2920296742201793
* 5 | 1 | 1.0 | 2.584962500721156  | 0.38685280723454163
* 6 | 0 | 0.0 | 2.807354922057604  | 0.0
*
* idcg = 14.595390756454922 (sum of last column)
*/
dcg = new DiscountedCumulativeGain(true, null, 10);
assertEquals(EXPECTED_NDCG, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
}
/**
* This tests metric when some documents in the search result don't have a
* rating provided by the user.
*
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1)
* -------------------------------------------------------------------------------------------
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 
* 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
* 3 | 3 | 7.0 | 2.0 | 3.5
* 4 | n/a | n/a | n/a | n/a
* 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163
* 6 | n/a | n/a | n/a | n/a
*
* dcg = 12.779642067948913 (sum of last column)
*/
public void testDCGAtSixMissingRatings() {
List<RatedDocument> rated = new ArrayList<>();
Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1 };
SearchHit[] hits = new SearchHit[6];
for (int i = 0; i < 6; i++) {
if (i < relevanceRatings.length) {
if (relevanceRatings[i] != null) {
rated.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
}
}
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
}
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
EvalQueryQuality result = dcg.evaluate("id", hits, rated);
assertEquals(12.779642067948913, result.getQualityLevel(), DELTA);
assertEquals(2, filterUnknownDocuments(result.getHitsAndRatings()).size());
/**
* Check with normalization: to get the maximal possible dcg, sort documents by
* relevance in descending order
*
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1)
* ----------------------------------------------------------------------------------------
* 1 | 3 | 7.0 | 1.0  | 7.0
* 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
* 3 | 2 | 3.0 | 2.0  | 1.5
* 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339
* 5 | n.a | n.a | n.a.  | n.a.
* 6 | n.a | n.a | n.a  | n.a
*
* idcg = 13.347184833073591 (sum of last column)
*/
dcg = new DiscountedCumulativeGain(true, null, 10);
assertEquals(12.779642067948913 / 13.347184833073591, dcg.evaluate("id", hits, rated).getQualityLevel(), DELTA);
}
/**
* This tests that normalization works as expected when there are more rated
* documents than search hits because we restrict DCG to be calculated at the
* fourth position
*
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1)
* -------------------------------------------------------------------------------------------
* 1 | 3 | 7.0 | 1.0 | 7.0 2 | 
* 2 | 3.0 | 1.5849625007211563 | 1.8927892607143721
* 3 | 3 | 7.0 | 2.0 | 3.5
* 4 | n/a | n/a | n/a | n/a
* -----------------------------------------------------------------
* 5 | 1 | 1.0 | 2.584962500721156 | 0.38685280723454163
* 6 | n/a | n/a | n/a | n/a
*
* dcg = 12.392789260714371 (sum of last column until position 4)
*/
public void testDCGAtFourMoreRatings() {
Integer[] relevanceRatings = new Integer[] { 3, 2, 3, null, 1, null };
List<RatedDocument> ratedDocs = new ArrayList<>();
for (int i = 0; i < 6; i++) {
if (i < relevanceRatings.length) {
if (relevanceRatings[i] != null) {
ratedDocs.add(new RatedDocument("index", Integer.toString(i), relevanceRatings[i]));
}
}
}
// only create four hits
SearchHit[] hits = new SearchHit[4];
for (int i = 0; i < 4; i++) {
hits[i] = new SearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap());
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
}
DiscountedCumulativeGain dcg = new DiscountedCumulativeGain();
EvalQueryQuality result = dcg.evaluate("id", hits, ratedDocs);
assertEquals(12.392789260714371, result.getQualityLevel(), DELTA);
assertEquals(1, filterUnknownDocuments(result.getHitsAndRatings()).size());
/**
* Check with normalization: to get the maximal possible dcg, sort documents by
* relevance in descending order
*
* rank | rel_rank | 2^(rel_rank) - 1 | log_2(rank + 1) | (2^(rel_rank) - 1) / log_2(rank + 1)
* ---------------------------------------------------------------------------------------
* 1 | 3 | 7.0 | 1.0  | 7.0
* 2 | 3 | 7.0 | 1.5849625007211563 | 4.416508275000202
* 3 | 2 | 3.0 | 2.0  | 1.5
* 4 | 1 | 1.0 | 2.321928094887362   | 0.43067655807339
* ---------------------------------------------------------------------------------------
* 5 | n.a | n.a | n.a.  | n.a.
* 6 | n.a | n.a | n.a  | n.a
*
* idcg = 13.347184833073591 (sum of last column)
*/
dcg = new DiscountedCumulativeGain(true, null, 10);
assertEquals(12.392789260714371 / 13.347184833073591, dcg.evaluate("id", hits, ratedDocs).getQualityLevel(), DELTA);
}
public void testParseFromXContent() throws IOException {
assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"normalize\": true, \"k\" : 15 }", 2, true, 15);
assertParsedCorrect("{ \"normalize\": false, \"k\" : 15 }", null, false, 15);
assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"k\" : 15 }", 2, false, 15);
assertParsedCorrect("{ \"unknown_doc_rating\": 2, \"normalize\": true }", 2, true, 10);
assertParsedCorrect("{ \"normalize\": true }", null, true, 10);
assertParsedCorrect("{ \"k\": 23 }", null, false, 23);
assertParsedCorrect("{ \"unknown_doc_rating\": 2 }", 2, false, 10);
}
private void assertParsedCorrect(String xContent, Integer expectedUnknownDocRating, boolean expectedNormalize, int expectedK)
throws IOException {
try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
DiscountedCumulativeGain dcgAt = DiscountedCumulativeGain.fromXContent(parser);
assertEquals(expectedUnknownDocRating, dcgAt.getUnknownDocRating());
assertEquals(expectedNormalize, dcgAt.getNormalize());
assertEquals(expectedK, dcgAt.getK());
}
}
public static DiscountedCumulativeGain createTestItem() {
boolean normalize = randomBoolean();
Integer unknownDocRating = new Integer(randomIntBetween(0, 1000));
return new DiscountedCumulativeGain(normalize, unknownDocRating, 10);
}
public void testXContentRoundtrip() throws IOException {
DiscountedCumulativeGain testItem = createTestItem();
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
itemParser.nextToken();
DiscountedCumulativeGain parsedItem = DiscountedCumulativeGain.fromXContent(itemParser);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}
public void testSerialization() throws IOException {
DiscountedCumulativeGain original = createTestItem();
DiscountedCumulativeGain deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()),
DiscountedCumulativeGain::new);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(createTestItem(), original -> {
return new DiscountedCumulativeGain(original.getNormalize(), original.getUnknownDocRating(), original.getK());
}, DiscountedCumulativeGainTests::mutateTestItem);
}
private static DiscountedCumulativeGain mutateTestItem(DiscountedCumulativeGain original) {
switch (randomIntBetween(0, 2)) {
case 0:
return new DiscountedCumulativeGain(!original.getNormalize(), original.getUnknownDocRating(), original.getK());
case 1:
return new DiscountedCumulativeGain(original.getNormalize(),
randomValueOtherThan(original.getUnknownDocRating(), () -> randomIntBetween(0, 10)), original.getK());
case 2:
return new DiscountedCumulativeGain(original.getNormalize(), original.getUnknownDocRating(),
randomValueOtherThan(original.getK(), () -> randomIntBetween(1, 10)));
default:
throw new IllegalArgumentException("mutation variant not allowed");
}
}
}

106
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/EvalQueryQualityTests.java Normal file
View File

@ -0,0 +1,106 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.index.rankeval.RatedDocument.DocumentKey;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class EvalQueryQualityTests extends ESTestCase {
private static NamedWriteableRegistry namedWritableRegistry = new NamedWriteableRegistry(new RankEvalPlugin().getNamedWriteables());
public static EvalQueryQuality randomEvalQueryQuality() {
List<DocumentKey> unknownDocs = new ArrayList<>();
int numberOfUnknownDocs = randomInt(5);
for (int i = 0; i < numberOfUnknownDocs; i++) {
unknownDocs.add(new DocumentKey(randomAlphaOfLength(10), randomAlphaOfLength(10)));
}
int numberOfSearchHits = randomInt(5);
List<RatedSearchHit> ratedHits = new ArrayList<>();
for (int i = 0; i < numberOfSearchHits; i++) {
ratedHits.add(RatedSearchHitTests.randomRatedSearchHit());
}
EvalQueryQuality evalQueryQuality = new EvalQueryQuality(randomAlphaOfLength(10),
randomDoubleBetween(0.0, 1.0, true));
if (randomBoolean()) {
if (randomBoolean()) {
evalQueryQuality.setMetricDetails(new PrecisionAtK.Breakdown(randomIntBetween(0, 1000), randomIntBetween(0, 1000)));
} else {
evalQueryQuality.setMetricDetails(new MeanReciprocalRank.Breakdown(randomIntBetween(0, 1000)));
}
}
evalQueryQuality.addHitsAndRatings(ratedHits);
return evalQueryQuality;
}
public void testSerialization() throws IOException {
EvalQueryQuality original = randomEvalQueryQuality();
EvalQueryQuality deserialized = copy(original);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
private static EvalQueryQuality copy(EvalQueryQuality original) throws IOException {
return ESTestCase.copyWriteable(original, namedWritableRegistry, EvalQueryQuality::new);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(randomEvalQueryQuality(), EvalQueryQualityTests::copy, EvalQueryQualityTests::mutateTestItem);
}
private static EvalQueryQuality mutateTestItem(EvalQueryQuality original) {
String id = original.getId();
double qualityLevel = original.getQualityLevel();
List<RatedSearchHit> ratedHits = new ArrayList<>(original.getHitsAndRatings());
MetricDetails metricDetails = original.getMetricDetails();
switch (randomIntBetween(0, 3)) {
case 0:
id = id + "_";
break;
case 1:
qualityLevel = qualityLevel + 0.1;
break;
case 2:
if (metricDetails == null) {
metricDetails = new PrecisionAtK.Breakdown(1, 5);
} else {
metricDetails = null;
}
break;
case 3:
ratedHits.add(RatedSearchHitTests.randomRatedSearchHit());
break;
default:
throw new IllegalStateException("The test should only allow four parameters mutated");
}
EvalQueryQuality evalQueryQuality = new EvalQueryQuality(id, qualityLevel);
evalQueryQuality.setMetricDetails(metricDetails);
evalQueryQuality.addHitsAndRatings(ratedHits);
return evalQueryQuality;
}
}

221
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/MeanReciprocalRankTests.java Normal file
View File

@ -0,0 +1,221 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.Index;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Vector;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class MeanReciprocalRankTests extends ESTestCase {
public void testParseFromXContent() throws IOException {
String xContent = "{ }";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
assertEquals(1, mrr.getRelevantRatingThreshold());
assertEquals(10, mrr.getK());
}
xContent = "{ \"relevant_rating_threshold\": 2 }";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
assertEquals(2, mrr.getRelevantRatingThreshold());
assertEquals(10, mrr.getK());
}
xContent = "{ \"relevant_rating_threshold\": 2, \"k\" : 15 }";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
assertEquals(2, mrr.getRelevantRatingThreshold());
assertEquals(15, mrr.getK());
}
xContent = "{ \"k\" : 15 }";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
MeanReciprocalRank mrr = MeanReciprocalRank.fromXContent(parser);
assertEquals(1, mrr.getRelevantRatingThreshold());
assertEquals(15, mrr.getK());
}
}
public void testMaxAcceptableRank() {
MeanReciprocalRank reciprocalRank = new MeanReciprocalRank();
int searchHits = randomIntBetween(1, 50);
SearchHit[] hits = createSearchHits(0, searchHits, "test");
List<RatedDocument> ratedDocs = new ArrayList<>();
int relevantAt = randomIntBetween(0, searchHits);
for (int i = 0; i <= searchHits; i++) {
if (i == relevantAt) {
ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.RELEVANT.ordinal()));
} else {
ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.IRRELEVANT.ordinal()));
}
}
int rankAtFirstRelevant = relevantAt + 1;
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
assertEquals(1.0 / rankAtFirstRelevant, evaluation.getQualityLevel(), Double.MIN_VALUE);
assertEquals(rankAtFirstRelevant, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
// check that if we have fewer search hits than relevant doc position,
// we don't find any result and get 0.0 quality level
reciprocalRank = new MeanReciprocalRank();
evaluation = reciprocalRank.evaluate("id", Arrays.copyOfRange(hits, 0, relevantAt), ratedDocs);
assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
}
public void testEvaluationOneRelevantInResults() {
MeanReciprocalRank reciprocalRank = new MeanReciprocalRank();
SearchHit[] hits = createSearchHits(0, 9, "test");
List<RatedDocument> ratedDocs = new ArrayList<>();
// mark one of the ten docs relevant
int relevantAt = randomIntBetween(0, 9);
for (int i = 0; i <= 20; i++) {
if (i == relevantAt) {
ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.RELEVANT.ordinal()));
} else {
ratedDocs.add(new RatedDocument("test", Integer.toString(i), TestRatingEnum.IRRELEVANT.ordinal()));
}
}
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
assertEquals(1.0 / (relevantAt + 1), evaluation.getQualityLevel(), Double.MIN_VALUE);
assertEquals(relevantAt + 1, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
}
/**
* test that the relevant rating threshold can be set to something larger than
* 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, so first
* relevant doc has third ranking position, so RR should be 1/3
*/
public void testPrecisionAtFiveRelevanceThreshold() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("test", "0", 0));
rated.add(new RatedDocument("test", "1", 1));
rated.add(new RatedDocument("test", "2", 2));
rated.add(new RatedDocument("test", "3", 3));
rated.add(new RatedDocument("test", "4", 4));
SearchHit[] hits = createSearchHits(0, 5, "test");
MeanReciprocalRank reciprocalRank = new MeanReciprocalRank(2, 10);
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, rated);
assertEquals((double) 1 / 3, evaluation.getQualityLevel(), 0.00001);
assertEquals(3, ((MeanReciprocalRank.Breakdown) evaluation.getMetricDetails()).getFirstRelevantRank());
}
public void testCombine() {
MeanReciprocalRank reciprocalRank = new MeanReciprocalRank();
Vector<EvalQueryQuality> partialResults = new Vector<>(3);
partialResults.add(new EvalQueryQuality("id1", 0.5));
partialResults.add(new EvalQueryQuality("id2", 1.0));
partialResults.add(new EvalQueryQuality("id3", 0.75));
assertEquals(0.75, reciprocalRank.combine(partialResults), Double.MIN_VALUE);
}
public void testEvaluationNoRelevantInResults() {
MeanReciprocalRank reciprocalRank = new MeanReciprocalRank();
SearchHit[] hits = createSearchHits(0, 9, "test");
List<RatedDocument> ratedDocs = new ArrayList<>();
EvalQueryQuality evaluation = reciprocalRank.evaluate("id", hits, ratedDocs);
assertEquals(0.0, evaluation.getQualityLevel(), Double.MIN_VALUE);
}
public void testXContentRoundtrip() throws IOException {
MeanReciprocalRank testItem = createTestItem();
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
itemParser.nextToken();
MeanReciprocalRank parsedItem = MeanReciprocalRank.fromXContent(itemParser);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}
/**
* Create SearchHits for testing, starting from dociId 'from' up to docId 'to'.
* The search hits index also need to be provided
*/
private static SearchHit[] createSearchHits(int from, int to, String index) {
SearchHit[] hits = new SearchHit[to + 1 - from];
for (int i = from; i <= to; i++) {
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap());
hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null));
}
return hits;
}
static MeanReciprocalRank createTestItem() {
return new MeanReciprocalRank(randomIntBetween(0, 20), randomIntBetween(1, 20));
}
public void testSerialization() throws IOException {
MeanReciprocalRank original = createTestItem();
MeanReciprocalRank deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()),
MeanReciprocalRank::new);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(createTestItem(), MeanReciprocalRankTests::copy, MeanReciprocalRankTests::mutate);
}
private static MeanReciprocalRank copy(MeanReciprocalRank testItem) {
return new MeanReciprocalRank(testItem.getRelevantRatingThreshold(), testItem.getK());
}
private static MeanReciprocalRank mutate(MeanReciprocalRank testItem) {
if (randomBoolean()) {
return new MeanReciprocalRank(testItem.getRelevantRatingThreshold() + 1, testItem.getK());
} else {
return new MeanReciprocalRank(testItem.getRelevantRatingThreshold(), testItem.getK() + 1);
}
}
public void testInvalidRelevantThreshold() {
expectThrows(IllegalArgumentException.class, () -> new MeanReciprocalRank(-1, 1));
}
public void testInvalidK() {
expectThrows(IllegalArgumentException.class, () -> new MeanReciprocalRank(1, -1));
}
}

235
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtKTests.java Normal file
View File

@ -0,0 +1,235 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.Index;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Vector;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class PrecisionAtKTests extends ESTestCase {
public void testPrecisionAtFiveCalculation() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal()));
EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated);
assertEquals(1, evaluated.getQualityLevel(), 0.00001);
assertEquals(1, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(1, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testPrecisionAtFiveIgnoreOneResult() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "1", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "2", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "3", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "4", TestRatingEnum.IRRELEVANT.ordinal()));
EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated, "test"), rated);
assertEquals((double) 4 / 5, evaluated.getQualityLevel(), 0.00001);
assertEquals(4, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(5, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
/**
* test that the relevant rating threshold can be set to something larger than
* 1. e.g. we set it to 2 here and expect dics 0-2 to be not relevant, doc 3 and
* 4 to be relevant
*/
public void testPrecisionAtFiveRelevanceThreshold() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(createRatedDoc("test", "0", 0));
rated.add(createRatedDoc("test", "1", 1));
rated.add(createRatedDoc("test", "2", 2));
rated.add(createRatedDoc("test", "3", 3));
rated.add(createRatedDoc("test", "4", 4));
PrecisionAtK precisionAtN = new PrecisionAtK(2, false, 5);
EvalQueryQuality evaluated = precisionAtN.evaluate("id", toSearchHits(rated, "test"), rated);
assertEquals((double) 3 / 5, evaluated.getQualityLevel(), 0.00001);
assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(5, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testPrecisionAtFiveCorrectIndex() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(createRatedDoc("test_other", "0", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test_other", "1", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "1", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "2", TestRatingEnum.IRRELEVANT.ordinal()));
// the following search hits contain only the last three documents
EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", toSearchHits(rated.subList(2, 5), "test"), rated);
assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001);
assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testIgnoreUnlabeled() {
List<RatedDocument> rated = new ArrayList<>();
rated.add(createRatedDoc("test", "0", TestRatingEnum.RELEVANT.ordinal()));
rated.add(createRatedDoc("test", "1", TestRatingEnum.RELEVANT.ordinal()));
// add an unlabeled search hit
SearchHit[] searchHits = Arrays.copyOf(toSearchHits(rated, "test"), 3);
searchHits[2] = new SearchHit(2, "2", new Text("testtype"), Collections.emptyMap());
searchHits[2].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", searchHits, rated);
assertEquals((double) 2 / 3, evaluated.getQualityLevel(), 0.00001);
assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(3, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
// also try with setting `ignore_unlabeled`
PrecisionAtK prec = new PrecisionAtK(1, true, 10);
evaluated = prec.evaluate("id", searchHits, rated);
assertEquals((double) 2 / 2, evaluated.getQualityLevel(), 0.00001);
assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(2, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testNoRatedDocs() throws Exception {
SearchHit[] hits = new SearchHit[5];
for (int i = 0; i < 5; i++) {
hits[i] = new SearchHit(i, i + "", new Text("type"), Collections.emptyMap());
hits[i].shard(new SearchShardTarget("testnode", new Index("index", "uuid"), 0, null));
}
EvalQueryQuality evaluated = (new PrecisionAtK()).evaluate("id", hits, Collections.emptyList());
assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(5, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
// also try with setting `ignore_unlabeled`
PrecisionAtK prec = new PrecisionAtK(1, true, 10);
evaluated = prec.evaluate("id", hits, Collections.emptyList());
assertEquals(0.0d, evaluated.getQualityLevel(), 0.00001);
assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRelevantRetrieved());
assertEquals(0, ((PrecisionAtK.Breakdown) evaluated.getMetricDetails()).getRetrieved());
}
public void testParseFromXContent() throws IOException {
String xContent = " {\n" + " \"relevant_rating_threshold\" : 2" + "}";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, xContent)) {
PrecisionAtK precicionAt = PrecisionAtK.fromXContent(parser);
assertEquals(2, precicionAt.getRelevantRatingThreshold());
}
}
public void testCombine() {
PrecisionAtK metric = new PrecisionAtK();
Vector<EvalQueryQuality> partialResults = new Vector<>(3);
partialResults.add(new EvalQueryQuality("a", 0.1));
partialResults.add(new EvalQueryQuality("b", 0.2));
partialResults.add(new EvalQueryQuality("c", 0.6));
assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE);
}
public void testInvalidRelevantThreshold() {
expectThrows(IllegalArgumentException.class, () -> new PrecisionAtK(-1, false, 10));
}
public void testInvalidK() {
expectThrows(IllegalArgumentException.class, () -> new PrecisionAtK(1, false, -10));
}
public static PrecisionAtK createTestItem() {
return new PrecisionAtK(randomIntBetween(0, 10), randomBoolean(), randomIntBetween(1, 50));
}
public void testXContentRoundtrip() throws IOException {
PrecisionAtK testItem = createTestItem();
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
itemParser.nextToken();
PrecisionAtK parsedItem = PrecisionAtK.fromXContent(itemParser);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}
public void testSerialization() throws IOException {
PrecisionAtK original = createTestItem();
PrecisionAtK deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()),
PrecisionAtK::new);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(createTestItem(), PrecisionAtKTests::copy, PrecisionAtKTests::mutate);
}
private static PrecisionAtK copy(PrecisionAtK original) {
return new PrecisionAtK(original.getRelevantRatingThreshold(), original.getIgnoreUnlabeled(), original.forcedSearchSize().get());
}
private static PrecisionAtK mutate(PrecisionAtK original) {
PrecisionAtK pAtK;
switch (randomIntBetween(0, 2)) {
case 0:
pAtK = new PrecisionAtK(original.getRelevantRatingThreshold(), !original.getIgnoreUnlabeled(),
original.forcedSearchSize().get());
break;
case 1:
pAtK = new PrecisionAtK(randomValueOtherThan(original.getRelevantRatingThreshold(), () -> randomIntBetween(0, 10)),
original.getIgnoreUnlabeled(), original.forcedSearchSize().get());
break;
case 2:
pAtK = new PrecisionAtK(original.getRelevantRatingThreshold(),
original.getIgnoreUnlabeled(), original.forcedSearchSize().get() + 1);
break;
default:
throw new IllegalStateException("The test should only allow three parameters mutated");
}
return pAtK;
}
private static SearchHit[] toSearchHits(List<RatedDocument> rated, String index) {
SearchHit[] hits = new SearchHit[rated.size()];
for (int i = 0; i < rated.size(); i++) {
hits[i] = new SearchHit(i, i + "", new Text(""), Collections.emptyMap());
hits[i].shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null));
}
return hits;
}
private static RatedDocument createRatedDoc(String index, String id, int rating) {
return new RatedDocument(index, id, rating);
}
}

266
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestIT.java Normal file
View File

@ -0,0 +1,266 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.ElasticsearchException;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
public class RankEvalRequestIT extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return Arrays.asList(RankEvalPlugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(RankEvalPlugin.class);
}
@Before
public void setup() {
createIndex("test");
ensureGreen();
client().prepareIndex("test", "testtype").setId("1")
.setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get();
client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get();
client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get();
refresh();
}
/**
* Test cases retrieves all six documents indexed above. The first part checks the Prec@10 calculation where
* all unlabeled docs are treated as "unrelevant". We average Prec@ metric across two search use cases, the
* first one that labels 4 out of the 6 documents as relevant, the second one with only one relevant document.
*/
public void testPrecisionAtRequest() {
List<RatedRequest> specifications = new ArrayList<>();
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
testQuery.sort("_id");
RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
createRelevant("2", "3", "4", "5"), testQuery);
amsterdamRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
specifications.add(amsterdamRequest);
RatedRequest berlinRequest = new RatedRequest("berlin_query", createRelevant("1"),
testQuery);
berlinRequest.addSummaryFields(Arrays.asList(new String[] { "text", "title" }));
specifications.add(berlinRequest);
PrecisionAtK metric = new PrecisionAtK(1, false, 10);
RankEvalSpec task = new RankEvalSpec(specifications, metric);
task.addIndices(Collections.singletonList("test"));
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(),
RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request())
.actionGet();
// the expected Prec@ for the first query is 4/6 and the expected Prec@ for the
// second is 1/6, divided by 2 to get the average
double expectedPrecision = (1.0 / 6.0 + 4.0 / 6.0) / 2.0;
assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
Set<Entry<String, EvalQueryQuality>> entrySet = response.getPartialResults().entrySet();
assertEquals(2, entrySet.size());
for (Entry<String, EvalQueryQuality> entry : entrySet) {
EvalQueryQuality quality = entry.getValue();
if (entry.getKey() == "amsterdam_query") {
assertEquals(2, filterUnknownDocuments(quality.getHitsAndRatings()).size());
List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
assertEquals(6, hitsAndRatings.size());
for (RatedSearchHit hit : hitsAndRatings) {
String id = hit.getSearchHit().getId();
if (id.equals("1") || id.equals("6")) {
assertFalse(hit.getRating().isPresent());
} else {
assertEquals(TestRatingEnum.RELEVANT.ordinal(), hit.getRating().get().intValue());
}
}
}
if (entry.getKey() == "berlin_query") {
assertEquals(5, filterUnknownDocuments(quality.getHitsAndRatings()).size());
List<RatedSearchHit> hitsAndRatings = quality.getHitsAndRatings();
assertEquals(6, hitsAndRatings.size());
for (RatedSearchHit hit : hitsAndRatings) {
String id = hit.getSearchHit().getId();
if (id.equals("1")) {
assertEquals(TestRatingEnum.RELEVANT.ordinal(), hit.getRating().get().intValue());
} else {
assertFalse(hit.getRating().isPresent());
}
}
}
}
// test that a different window size k affects the result
metric = new PrecisionAtK(1, false, 3);
task = new RankEvalSpec(specifications, metric);
task.addIndices(Collections.singletonList("test"));
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
// if we look only at top 3 documente, the expected P@3 for the first query is
// 2/3 and the expected Prec@ for the second is 1/3, divided by 2 to get the average
expectedPrecision = (1.0 / 3.0 + 2.0 / 3.0) / 2.0;
assertEquals(expectedPrecision, response.getEvaluationResult(), Double.MIN_VALUE);
}
/**
* This test assumes we are using the same ratings as in {@link DiscountedCumulativeGainTests#testDCGAt()}.
* See details in that test case for how the expected values are calculated
*/
public void testDCGRequest() {
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
testQuery.sort("_id");
List<RatedRequest> specifications = new ArrayList<>();
List<RatedDocument> ratedDocs = Arrays.asList(
new RatedDocument("test", "1", 3),
new RatedDocument("test", "2", 2),
new RatedDocument("test", "3", 3),
new RatedDocument("test", "4", 0),
new RatedDocument("test", "5", 1),
new RatedDocument("test", "6", 2));
specifications.add(new RatedRequest("amsterdam_query", ratedDocs, testQuery));
DiscountedCumulativeGain metric = new DiscountedCumulativeGain(false, null, 10);
RankEvalSpec task = new RankEvalSpec(specifications, metric);
task.addIndices(Collections.singletonList("test"));
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getEvaluationResult(), Double.MIN_VALUE);
// test that a different window size k affects the result
metric = new DiscountedCumulativeGain(false, null, 3);
task = new RankEvalSpec(specifications, metric);
task.addIndices(Collections.singletonList("test"));
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
assertEquals(12.392789260714371, response.getEvaluationResult(), Double.MIN_VALUE);
}
public void testMRRRequest() {
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
testQuery.sort("_id");
List<RatedRequest> specifications = new ArrayList<>();
specifications.add(new RatedRequest("amsterdam_query", createRelevant("5"), testQuery));
specifications.add(new RatedRequest("berlin_query", createRelevant("1"), testQuery));
MeanReciprocalRank metric = new MeanReciprocalRank(1, 10);
RankEvalSpec task = new RankEvalSpec(specifications, metric);
task.addIndices(Collections.singletonList("test"));
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
// the expected reciprocal rank for the amsterdam_query is 1/5
// the expected reciprocal rank for the berlin_query is 1/1
// dividing by 2 to get the average
double expectedMRR = (1.0 / 1.0 + 1.0 / 5.0) / 2.0;
assertEquals(expectedMRR, response.getEvaluationResult(), 0.0);
// test that a different window size k affects the result
metric = new MeanReciprocalRank(1, 3);
task = new RankEvalSpec(specifications, metric);
task.addIndices(Collections.singletonList("test"));
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
// limiting to top 3 results, the amsterdam_query has no relevant document in it
// the reciprocal rank for the berlin_query is 1/1
// dividing by 2 to get the average
expectedMRR = (1.0/ 1.0) / 2.0;
assertEquals(expectedMRR, response.getEvaluationResult(), 0.0);
}
/**
* test that running a bad query (e.g. one that will target a non existing
* field) will produce an error in the response
*/
public void testBadQuery() {
List<String> indices = Arrays.asList(new String[] { "test" });
List<RatedRequest> specifications = new ArrayList<>();
SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder();
amsterdamQuery.query(new MatchAllQueryBuilder());
RatedRequest amsterdamRequest = new RatedRequest("amsterdam_query",
createRelevant("2", "3", "4", "5"), amsterdamQuery);
specifications.add(amsterdamRequest);
SearchSourceBuilder brokenQuery = new SearchSourceBuilder();
brokenQuery.query(QueryBuilders.termQuery("population", "noStringOnNumericFields"));
RatedRequest brokenRequest = new RatedRequest("broken_query", createRelevant("1"),
brokenQuery);
specifications.add(brokenRequest);
RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
task.addIndices(indices);
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
assertEquals(1, response.getFailures().size());
ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(response.getFailures().get("broken_query"));
assertEquals("java.lang.NumberFormatException: For input string: \"noStringOnNumericFields\"", rootCauses[0].getCause().toString());
}
private static List<RatedDocument> createRelevant(String... docs) {
List<RatedDocument> relevant = new ArrayList<>();
for (String doc : docs) {
relevant.add(new RatedDocument("test", doc, TestRatingEnum.RELEVANT.ordinal()));
}
return relevant;
}
}

123
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalResponseTests.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.BytesStreamOutput;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentLocation;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.Index;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchShardTarget;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
public class RankEvalResponseTests extends ESTestCase {
private static RankEvalResponse createRandomResponse() {
int numberOfRequests = randomIntBetween(0, 5);
Map<String, EvalQueryQuality> partials = new HashMap<>(numberOfRequests);
for (int i = 0; i < numberOfRequests; i++) {
String id = randomAlphaOfLengthBetween(3, 10);
EvalQueryQuality evalQuality = new EvalQueryQuality(id,
randomDoubleBetween(0.0, 1.0, true));
int numberOfDocs = randomIntBetween(0, 5);
List<RatedSearchHit> ratedHits = new ArrayList<>(numberOfDocs);
for (int d = 0; d < numberOfDocs; d++) {
ratedHits.add(searchHit(randomAlphaOfLength(10), randomIntBetween(0, 1000), randomIntBetween(0, 10)));
}
evalQuality.addHitsAndRatings(ratedHits);
partials.put(id, evalQuality);
}
int numberOfErrors = randomIntBetween(0, 2);
Map<String, Exception> errors = new HashMap<>(numberOfRequests);
for (int i = 0; i < numberOfErrors; i++) {
errors.put(randomAlphaOfLengthBetween(3, 10),
new IllegalArgumentException(randomAlphaOfLength(10)));
}
return new RankEvalResponse(randomDouble(), partials, errors);
}
public void testSerialization() throws IOException {
RankEvalResponse randomResponse = createRandomResponse();
try (BytesStreamOutput output = new BytesStreamOutput()) {
randomResponse.writeTo(output);
try (StreamInput in = output.bytes().streamInput()) {
RankEvalResponse deserializedResponse = new RankEvalResponse();
deserializedResponse.readFrom(in);
assertEquals(randomResponse.getEvaluationResult(), deserializedResponse.getEvaluationResult(), Double.MIN_VALUE);
assertEquals(randomResponse.getPartialResults(), deserializedResponse.getPartialResults());
assertEquals(randomResponse.getFailures().keySet(), deserializedResponse.getFailures().keySet());
assertNotSame(randomResponse, deserializedResponse);
assertEquals(-1, in.read());
}
}
}
public void testToXContent() throws IOException {
EvalQueryQuality coffeeQueryQuality = new EvalQueryQuality("coffee_query", 0.1);
coffeeQueryQuality.addHitsAndRatings(Arrays.asList(searchHit("index", 123, 5), searchHit("index", 456, null)));
RankEvalResponse response = new RankEvalResponse(0.123, Collections.singletonMap("coffee_query", coffeeQueryQuality),
Collections.singletonMap("beer_query", new ParsingException(new XContentLocation(0, 0), "someMsg")));
XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
String xContent = response.toXContent(builder, ToXContent.EMPTY_PARAMS).bytes().utf8ToString();
assertEquals(("{" +
" \"rank_eval\": {" +
" \"quality_level\": 0.123," +
" \"details\": {" +
" \"coffee_query\": {" +
" \"quality_level\": 0.1," +
" \"unknown_docs\": [{\"_index\":\"index\",\"_id\":\"456\"}]," +
" \"hits\":[{\"hit\":{\"_index\":\"index\",\"_type\":\"\",\"_id\":\"123\",\"_score\":1.0}," +
" \"rating\":5}," +
" {\"hit\":{\"_index\":\"index\",\"_type\":\"\",\"_id\":\"456\",\"_score\":1.0}," +
" \"rating\":null}" +
" ]" +
" }" +
" }," +
" \"failures\": {" +
" \"beer_query\": {" +
" \"error\": \"ParsingException[someMsg]\"" +
" }" +
" }" +
" }" +
"}").replaceAll("\\s+", ""), xContent);
}
private static RatedSearchHit searchHit(String index, int docId, Integer rating) {
SearchHit hit = new SearchHit(docId, docId + "", new Text(""), Collections.emptyMap());
hit.shard(new SearchShardTarget("testnode", new Index(index, "uuid"), 0, null));
hit.score(1.0f);
return new RatedSearchHit(hit, rating != null ? Optional.of(rating) : Optional.empty());
}
}

205
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java Normal file
View File

@ -0,0 +1,205 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.function.Supplier;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class RankEvalSpecTests extends ESTestCase {
@SuppressWarnings("resource")
@Override
protected NamedXContentRegistry xContentRegistry() {
return new NamedXContentRegistry(new RankEvalPlugin().getNamedXContent());
}
private static <T> List<T> randomList(Supplier<T> randomSupplier) {
List<T> result = new ArrayList<>();
int size = randomIntBetween(1, 20);
for (int i = 0; i < size; i++) {
result.add(randomSupplier.get());
}
return result;
}
private static RankEvalSpec createTestItem() throws IOException {
Supplier<EvaluationMetric> metric = randomFrom(Arrays.asList(
() -> PrecisionAtKTests.createTestItem(),
() -> MeanReciprocalRankTests.createTestItem(),
() -> DiscountedCumulativeGainTests.createTestItem()));
List<RatedRequest> ratedRequests = null;
Collection<ScriptWithId> templates = null;
if (randomBoolean()) {
final Map<String, Object> params = randomBoolean() ? Collections.emptyMap() : Collections.singletonMap("key", "value");
String script;
try (XContentBuilder builder = XContentFactory.jsonBuilder()) {
builder.startObject();
builder.field("field", randomAlphaOfLengthBetween(1, 5));
builder.endObject();
script = builder.string();
}
templates = new HashSet<>();
templates.add(new ScriptWithId("templateId", new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, script, params)));
Map<String, Object> templateParams = new HashMap<>();
templateParams.put("key", "value");
RatedRequest ratedRequest = new RatedRequest("id", Arrays.asList(RatedDocumentTests.createRatedDocument()), templateParams,
"templateId");
ratedRequests = Arrays.asList(ratedRequest);
} else {
RatedRequest ratedRequest = new RatedRequest("id", Arrays.asList(RatedDocumentTests.createRatedDocument()),
new SearchSourceBuilder());
ratedRequests = Arrays.asList(ratedRequest);
}
RankEvalSpec spec = new RankEvalSpec(ratedRequests, metric.get(), templates);
maybeSet(spec::setMaxConcurrentSearches, randomInt(100));
List<String> indices = new ArrayList<>();
int size = randomIntBetween(0, 20);
for (int i = 0; i < size; i++) {
indices.add(randomAlphaOfLengthBetween(0, 50));
}
spec.addIndices(indices);
return spec;
}
public void testXContentRoundtrip() throws IOException {
RankEvalSpec testItem = createTestItem();
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(XContentFactory.jsonBuilder(), ToXContent.EMPTY_PARAMS));
try (XContentParser parser = createParser(JsonXContent.jsonXContent, shuffled.bytes())) {
RankEvalSpec parsedItem = RankEvalSpec.parse(parser);
// indices, come from URL parameters, so they don't survive xContent roundtrip
// for the sake of being able to use equals() next, we add it to the parsed object
parsedItem.addIndices(testItem.getIndices());
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}
public void testSerialization() throws IOException {
RankEvalSpec original = createTestItem();
RankEvalSpec deserialized = copy(original);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
private static RankEvalSpec copy(RankEvalSpec original) throws IOException {
List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, PrecisionAtK.NAME, PrecisionAtK::new));
namedWriteables.add(
new NamedWriteableRegistry.Entry(EvaluationMetric.class, DiscountedCumulativeGain.NAME, DiscountedCumulativeGain::new));
namedWriteables.add(new NamedWriteableRegistry.Entry(EvaluationMetric.class, MeanReciprocalRank.NAME, MeanReciprocalRank::new));
return ESTestCase.copyWriteable(original, new NamedWriteableRegistry(namedWriteables), RankEvalSpec::new);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(createTestItem(), RankEvalSpecTests::copy, RankEvalSpecTests::mutateTestItem);
}
private static RankEvalSpec mutateTestItem(RankEvalSpec original) {
List<RatedRequest> ratedRequests = new ArrayList<>(original.getRatedRequests());
EvaluationMetric metric = original.getMetric();
Map<String, Script> templates = new HashMap<>(original.getTemplates());
List<String> indices = new ArrayList<>(original.getIndices());
int mutate = randomIntBetween(0, 3);
switch (mutate) {
case 0:
RatedRequest request = RatedRequestsTests.createTestItem(true);
ratedRequests.add(request);
break;
case 1:
if (metric instanceof PrecisionAtK) {
metric = new DiscountedCumulativeGain();
} else {
metric = new PrecisionAtK();
}
break;
case 2:
templates.put("mutation", new Script(ScriptType.INLINE, "mustache", randomAlphaOfLength(10), new HashMap<>()));
break;
case 3:
indices.add(randomAlphaOfLength(5));
break;
default:
throw new IllegalStateException("Requested to modify more than available parameters.");
}
List<ScriptWithId> scripts = new ArrayList<>();
for (Entry<String, Script> entry : templates.entrySet()) {
scripts.add(new ScriptWithId(entry.getKey(), entry.getValue()));
}
RankEvalSpec result = new RankEvalSpec(ratedRequests, metric, scripts);
result.addIndices(indices);
return result;
}
public void testMissingRatedRequestsFails() {
EvaluationMetric metric = new PrecisionAtK();
expectThrows(IllegalArgumentException.class, () -> new RankEvalSpec(new ArrayList<>(), metric));
expectThrows(IllegalArgumentException.class, () -> new RankEvalSpec(null, metric));
}
public void testMissingMetricFails() {
List<RatedRequest> ratedRequests = randomList(() -> RatedRequestsTests.createTestItem(randomBoolean()));
expectThrows(NullPointerException.class, () -> new RankEvalSpec(ratedRequests, null));
}
public void testMissingTemplateAndSearchRequestFails() {
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
Map<String, Object> params = new HashMap<>();
params.put("key", "value");
RatedRequest request = new RatedRequest("id", ratedDocs, params, "templateId");
List<RatedRequest> ratedRequests = Arrays.asList(request);
expectThrows(IllegalStateException.class, () -> new RankEvalSpec(ratedRequests, new PrecisionAtK()));
}
}

37
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalYamlIT.java Normal file
View File

@ -0,0 +1,37 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
public class RankEvalYamlIT extends ESClientYamlSuiteTestCase {
public RankEvalYamlIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
}

88
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java Normal file
View File

@ -0,0 +1,88 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.Collections;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class RatedDocumentTests extends ESTestCase {
public static RatedDocument createRatedDocument() {
return new RatedDocument(randomAlphaOfLength(10), randomAlphaOfLength(10), randomInt());
}
public void testXContentParsing() throws IOException {
RatedDocument testItem = createRatedDocument();
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
RatedDocument parsedItem = RatedDocument.fromXContent(itemParser);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}
public void testSerialization() throws IOException {
RatedDocument original = createRatedDocument();
RatedDocument deserialized = ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()),
RatedDocument::new);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(createRatedDocument(), original -> {
return new RatedDocument(original.getIndex(), original.getDocID(), original.getRating());
}, RatedDocumentTests::mutateTestItem);
}
private static RatedDocument mutateTestItem(RatedDocument original) {
int rating = original.getRating();
String index = original.getIndex();
String docId = original.getDocID();
switch (randomIntBetween(0, 2)) {
case 0:
rating = randomValueOtherThan(rating, () -> randomInt());
break;
case 1:
index = randomValueOtherThan(index, () -> randomAlphaOfLength(10));
break;
case 2:
docId = randomValueOtherThan(docId, () -> randomAlphaOfLength(10));
break;
default:
throw new IllegalStateException("The test should only allow two parameters mutated");
}
return new RatedDocument(index, docId, rating);
}
}

282
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java Normal file
View File

@ -0,0 +1,282 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Stream;
import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.toList;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class RatedRequestsTests extends ESTestCase {
private static NamedXContentRegistry xContentRegistry;
@BeforeClass
public static void init() {
xContentRegistry = new NamedXContentRegistry(
Stream.of(new SearchModule(Settings.EMPTY, false, emptyList()).getNamedXContents().stream()).flatMap(Function.identity())
.collect(toList()));
}
@AfterClass
public static void afterClass() throws Exception {
xContentRegistry = null;
}
@Override
protected NamedXContentRegistry xContentRegistry() {
return xContentRegistry;
}
public static RatedRequest createTestItem(boolean forceRequest) {
String requestId = randomAlphaOfLength(50);
List<RatedDocument> ratedDocs = new ArrayList<>();
int size = randomIntBetween(0, 2);
for (int i = 0; i < size; i++) {
ratedDocs.add(RatedDocumentTests.createRatedDocument());
}
Map<String, Object> params = new HashMap<>();
SearchSourceBuilder testRequest = null;
if (randomBoolean() || forceRequest) {
testRequest = new SearchSourceBuilder();
testRequest.size(randomIntBetween(0, Integer.MAX_VALUE));
testRequest.query(new MatchAllQueryBuilder());
} else {
int randomSize = randomIntBetween(1, 10);
for (int i = 0; i < randomSize; i++) {
params.put(randomAlphaOfLengthBetween(1, 10), randomAlphaOfLengthBetween(1, 10));
}
}
List<String> summaryFields = new ArrayList<>();
int numSummaryFields = randomIntBetween(0, 5);
for (int i = 0; i < numSummaryFields; i++) {
summaryFields.add(randomAlphaOfLength(5));
}
RatedRequest ratedRequest = null;
if (params.size() == 0) {
ratedRequest = new RatedRequest(requestId, ratedDocs, testRequest);
ratedRequest.addSummaryFields(summaryFields);
} else {
ratedRequest = new RatedRequest(requestId, ratedDocs, params, randomAlphaOfLength(5));
ratedRequest.addSummaryFields(summaryFields);
}
return ratedRequest;
}
public void testXContentRoundtrip() throws IOException {
RatedRequest testItem = createTestItem(randomBoolean());
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
XContentBuilder shuffled = shuffleXContent(testItem.toXContent(builder, ToXContent.EMPTY_PARAMS));
try (XContentParser itemParser = createParser(shuffled)) {
itemParser.nextToken();
RatedRequest parsedItem = RatedRequest.fromXContent(itemParser);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}
public void testSerialization() throws IOException {
RatedRequest original = createTestItem(randomBoolean());
RatedRequest deserialized = copy(original);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
private static RatedRequest copy(RatedRequest original) throws IOException {
List<NamedWriteableRegistry.Entry> namedWriteables = new ArrayList<>();
namedWriteables.add(new NamedWriteableRegistry.Entry(QueryBuilder.class, MatchAllQueryBuilder.NAME, MatchAllQueryBuilder::new));
return ESTestCase.copyWriteable(original, new NamedWriteableRegistry(namedWriteables), RatedRequest::new);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(createTestItem(randomBoolean()), RatedRequestsTests::copy, RatedRequestsTests::mutateTestItem);
}
private static RatedRequest mutateTestItem(RatedRequest original) {
String id = original.getId();
SearchSourceBuilder testRequest = original.getTestRequest();
List<RatedDocument> ratedDocs = original.getRatedDocs();
Map<String, Object> params = original.getParams();
List<String> summaryFields = original.getSummaryFields();
String templateId = original.getTemplateId();
int mutate = randomIntBetween(0, 3);
switch (mutate) {
case 0:
id = randomValueOtherThan(id, () -> randomAlphaOfLength(10));
break;
case 1:
if (testRequest != null) {
int size = randomValueOtherThan(testRequest.size(), () -> randomInt(Integer.MAX_VALUE));
testRequest = new SearchSourceBuilder();
testRequest.size(size);
testRequest.query(new MatchAllQueryBuilder());
} else {
if (randomBoolean()) {
Map<String, Object> mutated = new HashMap<>();
mutated.putAll(params);
mutated.put("one_more_key", "one_more_value");
params = mutated;
} else {
templateId = randomValueOtherThan(templateId, () -> randomAlphaOfLength(5));
}
}
break;
case 2:
ratedDocs = Arrays.asList(randomValueOtherThanMany(ratedDocs::contains, () -> RatedDocumentTests.createRatedDocument()));
break;
case 3:
summaryFields = Arrays.asList(randomValueOtherThanMany(summaryFields::contains, () -> randomAlphaOfLength(10)));
break;
default:
throw new IllegalStateException("Requested to modify more than available parameters.");
}
RatedRequest ratedRequest;
if (testRequest == null) {
ratedRequest = new RatedRequest(id, ratedDocs, params, templateId);
} else {
ratedRequest = new RatedRequest(id, ratedDocs, testRequest);
}
ratedRequest.addSummaryFields(summaryFields);
return ratedRequest;
}
public void testDuplicateRatedDocThrowsException() {
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1), new RatedDocument("index1", "id1", 5));
IllegalArgumentException ex = expectThrows(IllegalArgumentException.class,
() -> new RatedRequest("test_query", ratedDocs, new SearchSourceBuilder()));
assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}] in evaluation request [test_query]",
ex.getMessage());
Map<String, Object> params = new HashMap<>();
params.put("key", "value");
ex = expectThrows(IllegalArgumentException.class, () -> new RatedRequest("test_query", ratedDocs, params, "templateId"));
assertEquals("Found duplicate rated document key [{\"_index\":\"index1\",\"_id\":\"id1\"}] in evaluation request [test_query]",
ex.getMessage());
}
public void testNullSummaryFieldsTreatment() {
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder());
expectThrows(NullPointerException.class, () -> request.addSummaryFields(null));
}
public void testNullParamsTreatment() {
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
RatedRequest request = new RatedRequest("id", ratedDocs, new SearchSourceBuilder());
assertNotNull(request.getParams());
assertEquals(0, request.getParams().size());
}
public void testSettingNeitherParamsNorRequestThrows() {
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, null));
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, new HashMap<>(), "templateId"));
}
public void testSettingParamsWithoutTemplateIdThrows() {
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
Map<String, Object> params = new HashMap<>();
params.put("key", "value");
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, params, null));
}
public void testSettingTemplateIdNoParamsThrows() {
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
expectThrows(IllegalArgumentException.class, () -> new RatedRequest("id", ratedDocs, null, "templateId"));
}
/**
* test that modifying the order of index/docId to make sure it doesn't
* matter for parsing xContent
*/
public void testParseFromXContent() throws IOException {
String querySpecString = " {\n"
+ " \"id\": \"my_qa_query\",\n"
+ " \"request\": {\n"
+ " \"query\": {\n"
+ " \"bool\": {\n"
+ " \"must\": [\n"
+ " {\"match\": {\"beverage\": \"coffee\"}},\n"
+ " {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n"
+ " {\"term\": {\"time_of_day\": "
+ " {\"value\": \"morning\",\"boost\": 2}}},\n"
+ " {\"term\": {\"ip_location\": "
+ " {\"value\": \"ams\",\"boost\": 10}}}]}\n"
+ " },\n"
+ " \"size\": 10\n"
+ " },\n"
+ " \"summary_fields\" : [\"title\"],\n"
+ " \"ratings\": [\n"
+ " {\"_index\": \"test\" , \"_id\": \"1\", \"rating\" : 1 },\n"
+ " {\"_index\": \"test\", \"rating\" : 0, \"_id\": \"2\"},\n"
+ " {\"_id\": \"3\", \"_index\": \"test\", \"rating\" : 1} ]"
+ "}\n";
try (XContentParser parser = createParser(JsonXContent.jsonXContent, querySpecString)) {
RatedRequest specification = RatedRequest.fromXContent(parser);
assertEquals("my_qa_query", specification.getId());
assertNotNull(specification.getTestRequest());
List<RatedDocument> ratedDocs = specification.getRatedDocs();
assertEquals(3, ratedDocs.size());
for (int i = 0; i < 3; i++) {
assertEquals("" + (i + 1), ratedDocs.get(i).getDocID());
assertEquals("test", ratedDocs.get(i).getIndex());
if (i == 1) {
assertEquals(0, ratedDocs.get(i).getRating());
} else {
assertEquals(1, ratedDocs.get(i).getRating());
}
}
}
}
}

77
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedSearchHitTests.java Normal file
View File

@ -0,0 +1,77 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.Collections;
import java.util.Optional;
import static org.elasticsearch.test.EqualsHashCodeTestUtils.checkEqualsAndHashCode;
public class RatedSearchHitTests extends ESTestCase {
public static RatedSearchHit randomRatedSearchHit() {
Optional<Integer> rating = randomBoolean() ? Optional.empty()
: Optional.of(randomIntBetween(0, 5));
SearchHit searchHit = new SearchHit(randomIntBetween(0, 10), randomAlphaOfLength(10),
new Text(randomAlphaOfLength(10)), Collections.emptyMap());
RatedSearchHit ratedSearchHit = new RatedSearchHit(searchHit, rating);
return ratedSearchHit;
}
private static RatedSearchHit mutateTestItem(RatedSearchHit original) {
Optional<Integer> rating = original.getRating();
SearchHit hit = original.getSearchHit();
switch (randomIntBetween(0, 1)) {
case 0:
rating = rating.isPresent() ? Optional.of(rating.get() + 1) : Optional.of(randomInt(5));
break;
case 1:
hit = new SearchHit(hit.docId(), hit.getId() + randomAlphaOfLength(10),
new Text(hit.getType()), Collections.emptyMap());
break;
default:
throw new IllegalStateException("The test should only allow two parameters mutated");
}
return new RatedSearchHit(hit, rating);
}
public void testSerialization() throws IOException {
RatedSearchHit original = randomRatedSearchHit();
RatedSearchHit deserialized = copy(original);
assertEquals(deserialized, original);
assertEquals(deserialized.hashCode(), original.hashCode());
assertNotSame(deserialized, original);
}
public void testEqualsAndHash() throws IOException {
checkEqualsAndHashCode(randomRatedSearchHit(), RatedSearchHitTests::copy, RatedSearchHitTests::mutateTestItem);
}
private static RatedSearchHit copy(RatedSearchHit original) throws IOException {
return ESTestCase.copyWriteable(original, new NamedWriteableRegistry(Collections.emptyList()), RatedSearchHit::new);
}
}

24
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/TestRatingEnum.java Normal file
View File

@ -0,0 +1,24 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
enum TestRatingEnum {
IRRELEVANT, RELEVANT;
}

154
modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yml Normal file
View File

@ -0,0 +1,154 @@
---
"Response format":
- do:
indices.create:
index: foo
body:
settings:
index:
number_of_shards: 1
- do:
index:
index: foo
type: bar
id: doc1
body: { "text": "berlin" }
- do:
index:
index: foo
type: bar
id: doc2
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: doc3
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: doc4
body: { "text": "something about amsterdam and berlin" }
- do:
indices.refresh: {}
- do:
rank_eval:
body: {
"requests" : [
{
"id": "amsterdam_query",
"request": { "query": { "match" : {"text" : "amsterdam" }}},
"ratings": [
{"_index": "foo", "_id": "doc1", "rating": 0},
{"_index": "foo", "_id": "doc2", "rating": 1},
{"_index": "foo", "_id": "doc3", "rating": 1}]
},
{
"id" : "berlin_query",
"request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
}
],
"metric" : { "precision": { "ignore_unlabeled" : true }}
}
- match: { rank_eval.quality_level: 1}
- match: { rank_eval.details.amsterdam_query.quality_level: 1.0}
- match: { rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_id": "doc4"}]}
- match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 2, "docs_retrieved": 2}}
- length: { rank_eval.details.amsterdam_query.hits: 3}
- match: { rank_eval.details.amsterdam_query.hits.0.hit._id: "doc2"}
- match: { rank_eval.details.amsterdam_query.hits.0.rating: 1}
- match: { rank_eval.details.amsterdam_query.hits.1.hit._id: "doc3"}
- match: { rank_eval.details.amsterdam_query.hits.1.rating: 1}
- match: { rank_eval.details.amsterdam_query.hits.2.hit._id: "doc4"}
- is_false: rank_eval.details.amsterdam_query.hits.2.rating
- match: { rank_eval.details.berlin_query.quality_level: 1.0}
- match: { rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_id": "doc4"}]}
- match: { rank_eval.details.berlin_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
- length: { rank_eval.details.berlin_query.hits: 2}
- match: { rank_eval.details.berlin_query.hits.0.hit._id: "doc1" }
- match: { rank_eval.details.berlin_query.hits.0.rating: 1}
- match: { rank_eval.details.berlin_query.hits.1.hit._id: "doc4" }
- is_false: rank_eval.details.berlin_query.hits.1.rating
---
"Mean Reciprocal Rank":
- do:
indices.create:
index: foo
body:
settings:
index:
number_of_shards: 1
- do:
index:
index: foo
type: bar
id: doc1
body: { "text": "berlin" }
- do:
index:
index: foo
type: bar
id: doc2
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: doc3
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: doc4
body: { "text": "something about amsterdam and berlin" }
- do:
indices.refresh: {}
- do:
rank_eval:
body: {
"requests" : [
{
"id": "amsterdam_query",
"request": { "query": { "match" : {"text" : "amsterdam" }}},
# doc4 should be returned in third position, so reciprocal rank is 1/3
"ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
},
{
"id" : "berlin_query",
"request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
# doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2
"ratings": [{"_index": "foo", "_id": "doc4", "rating": 1}]
}
],
"metric" : { "mean_reciprocal_rank": {} }
}
# average is (1/3 + 1/2)/2 = 5/12 ~ 0.41666666666666663
- match: {rank_eval.quality_level: 0.41666666666666663}
- match: {rank_eval.details.amsterdam_query.quality_level: 0.3333333333333333}
- match: {rank_eval.details.amsterdam_query.metric_details: {"first_relevant": 3}}
- match: {rank_eval.details.amsterdam_query.unknown_docs: [ {"_index": "foo", "_id": "doc2"},
{"_index": "foo", "_id": "doc3"} ]}
- match: {rank_eval.details.berlin_query.quality_level: 0.5}
- match: {rank_eval.details.berlin_query.metric_details: {"first_relevant": 2}}
- match: {rank_eval.details.berlin_query.unknown_docs: [ {"_index": "foo", "_id": "doc1"}]}

133
modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/20_dcg.yml Normal file
View File

@ -0,0 +1,133 @@
---
"Response format":
- do:
index:
index: foo
type: bar
id: doc1
body: { "bar": 1 }
- do:
index:
index: foo
type: bar
id: doc2
body: { "bar": 2 }
- do:
index:
index: foo
type: bar
id: doc3
body: { "bar": 3 }
- do:
index:
index: foo
type: bar
id: doc4
body: { "bar": 4 }
- do:
index:
index: foo
type: bar
id: doc5
body: { "bar": 5 }
- do:
index:
index: foo
type: bar
id: doc6
body: { "bar": 6 }
- do:
indices.refresh: {}
- do:
rank_eval:
body: {
"requests" : [
{
"id": "dcg_query",
"request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] },
"ratings": [
{"_index" : "foo", "_id" : "doc1", "rating": 3},
{"_index" : "foo", "_id" : "doc2", "rating": 2},
{"_index" : "foo", "_id" : "doc3", "rating": 3},
{"_index" : "foo", "_id" : "doc4", "rating": 0},
{"_index" : "foo", "_id" : "doc5", "rating": 1},
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
}
],
"metric" : { "dcg": {}}
}
- match: {rank_eval.quality_level: 13.84826362927298}
- match: {rank_eval.details.dcg_query.quality_level: 13.84826362927298}
- match: {rank_eval.details.dcg_query.unknown_docs: [ ]}
# reverse the order in which the results are returned (less relevant docs first)
- do:
rank_eval:
body: {
"requests" : [
{
"id": "dcg_query_reverse",
"request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] },
"ratings": [
{"_index" : "foo", "_id" : "doc1", "rating": 3},
{"_index" : "foo", "_id" : "doc2", "rating": 2},
{"_index" : "foo", "_id" : "doc3", "rating": 3},
{"_index" : "foo", "_id" : "doc4", "rating": 0},
{"_index" : "foo", "_id" : "doc5", "rating": 1},
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
},
],
"metric" : { "dcg": { }}
}
- match: {rank_eval.quality_level: 10.29967439154499}
- match: {rank_eval.details.dcg_query_reverse.quality_level: 10.29967439154499}
- match: {rank_eval.details.dcg_query_reverse.unknown_docs: [ ]}
# if we mix both, we should get the average
- do:
rank_eval:
body: {
"requests" : [
{
"id": "dcg_query",
"request": { "query": { "match_all" : {}}, "sort" : [ "bar" ] },
"ratings": [
{"_index" : "foo", "_id" : "doc1", "rating": 3},
{"_index" : "foo", "_id" : "doc2", "rating": 2},
{"_index" : "foo", "_id" : "doc3", "rating": 3},
{"_index" : "foo", "_id" : "doc4", "rating": 0},
{"_index" : "foo", "_id" : "doc5", "rating": 1},
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
},
{
"id": "dcg_query_reverse",
"request": { "query": { "match_all" : {}}, "sort" : [ {"bar" : "desc" }] },
"ratings": [
{"_index" : "foo", "_id" : "doc1", "rating": 3},
{"_index" : "foo", "_id" : "doc2", "rating": 2},
{"_index" : "foo", "_id" : "doc3", "rating": 3},
{"_index" : "foo", "_id" : "doc4", "rating": 0},
{"_index" : "foo", "_id" : "doc5", "rating": 1},
{"_index" : "foo", "_id" : "doc6", "rating": 2}]
},
],
"metric" : { "dcg": { }}
}
- match: {rank_eval.quality_level: 12.073969010408984}
- match: {rank_eval.details.dcg_query.quality_level: 13.84826362927298}
- match: {rank_eval.details.dcg_query.unknown_docs: [ ]}
- match: {rank_eval.details.dcg_query_reverse.quality_level: 10.29967439154499}
- match: {rank_eval.details.dcg_query_reverse.unknown_docs: [ ]}

38
modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/30_failures.yml Normal file
View File

@ -0,0 +1,38 @@
---
"Response format":
- do:
index:
index: foo
type: bar
id: doc1
body: { "bar": 1 }
- do:
indices.refresh: {}
- do:
rank_eval:
body: {
"requests" : [
{
"id": "amsterdam_query",
"request": { "query": { "match_all" : { }}},
"ratings": [
{"_index": "foo", "_id": "doc1", "rating": 1}]
},
{
"id" : "invalid_query",
"request": { "query": { "range" : { "bar" : { "from" : "Basel", "time_zone": "+01:00" }}}},
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
}
],
"metric" : { "precision": { "ignore_unlabeled" : true }}
}
- match: { rank_eval.quality_level: 1}
- match: { rank_eval.details.amsterdam_query.quality_level: 1.0}
- match: { rank_eval.details.amsterdam_query.unknown_docs: [ ]}
- match: { rank_eval.details.amsterdam_query.metric_details: {"relevant_docs_retrieved": 1, "docs_retrieved": 1}}
- is_true: rank_eval.failures.invalid_query

28
qa/smoke-test-rank-eval-with-mustache/build.gradle Normal file
View File

@ -0,0 +1,28 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
apply plugin: 'elasticsearch.standalone-rest-test'
apply plugin: 'elasticsearch.rest-test'
dependencies {
testCompile project(path: ':modules:rank-eval', configuration: 'runtime')
testCompile project(path: ':modules:lang-mustache', configuration: 'runtime')
}

123
qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeMultipleTemplatesIT.java Normal file
View File

@ -0,0 +1,123 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
public class SmokeMultipleTemplatesIT extends ESIntegTestCase {
private static final String MATCH_TEMPLATE = "match_template";
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return Arrays.asList(RankEvalPlugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return Arrays.asList(RankEvalPlugin.class);
}
@Before
public void setup() {
createIndex("test");
ensureGreen();
client().prepareIndex("test", "testtype").setId("1")
.setSource("text", "berlin", "title", "Berlin, Germany").get();
client().prepareIndex("test", "testtype").setId("2")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("3")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("4")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("5")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("6")
.setSource("text", "amsterdam").get();
refresh();
}
public void testPrecisionAtRequest() throws IOException {
List<String> indices = Arrays.asList(new String[] { "test" });
List<RatedRequest> specifications = new ArrayList<>();
Map<String, Object> ams_params = new HashMap<>();
ams_params.put("querystring", "amsterdam");
RatedRequest amsterdamRequest = new RatedRequest(
"amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params, MATCH_TEMPLATE);
specifications.add(amsterdamRequest);
Map<String, Object> berlin_params = new HashMap<>();
berlin_params.put("querystring", "berlin");
RatedRequest berlinRequest = new RatedRequest(
"berlin_query", createRelevant("1"), berlin_params, MATCH_TEMPLATE);
specifications.add(berlinRequest);
PrecisionAtK metric = new PrecisionAtK();
ScriptWithId template =
new ScriptWithId(
MATCH_TEMPLATE,
new Script(
ScriptType.INLINE,
"mustache", "{\"query\": {\"match\": {\"text\": \"{{querystring}}\"}}}",
new HashMap<>()));
Set<ScriptWithId> templates = new HashSet<>();
templates.add(template);
RankEvalSpec task = new RankEvalSpec(specifications, metric, templates);
task.addIndices(indices);
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
builder.setRankEvalSpec(task);
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
assertEquals(0.9, response.getEvaluationResult(), Double.MIN_VALUE);
}
private static List<RatedDocument> createRelevant(String... docs) {
List<RatedDocument> relevant = new ArrayList<>();
for (String doc : docs) {
relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal()));
}
return relevant;
}
public enum Rating {
IRRELEVANT, RELEVANT;
}
}

39
qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/index/rankeval/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java Normal file
View File

@ -0,0 +1,39 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import com.carrotsearch.randomizedtesting.annotations.Name;
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
public class SmokeTestRankEvalWithMustacheYAMLTestSuiteIT extends ESClientYamlSuiteTestCase {
public SmokeTestRankEvalWithMustacheYAMLTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
super(testCandidate);
}
@ParametersFactory
public static Iterable<Object[]> parameters() throws Exception {
return ESClientYamlSuiteTestCase.createParameters();
}
}

67
qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yml Normal file
View File

@ -0,0 +1,67 @@
---
"Template request":
- do:
indices.create:
index: foo
body:
settings:
index:
number_of_shards: 1
- do:
index:
index: foo
type: bar
id: doc1
body: { "text": "berlin" }
- do:
index:
index: foo
type: bar
id: doc2
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: doc3
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: doc4
body: { "text": "something about amsterdam and berlin" }
- do:
indices.refresh: {}
- do:
rank_eval:
body: {
"templates": [ { "id": "match", "template": {"source": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ],
"requests" : [
{
"id": "amsterdam_query",
"params": { "query_string": "amsterdam" },
"template_id": "match",
"ratings": [
{"_index": "foo", "_id": "doc1", "rating": 0},
{"_index": "foo", "_id": "doc2", "rating": 1},
{"_index": "foo", "_id": "doc3", "rating": 1}]
},
{
"id" : "berlin_query",
"params": { "query_string": "berlin" },
"template_id": "match",
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
}
],
"metric" : { "precision": { }}
}
- match: {rank_eval.quality_level: 0.5833333333333333}
- match: {rank_eval.details.berlin_query.unknown_docs.0._id: "doc4"}
- match: {rank_eval.details.amsterdam_query.unknown_docs.0._id: "doc4"}

26
rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json Normal file
View File

@ -0,0 +1,26 @@
{
"rank_eval": {
"documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-rank-eval.html",
"methods": ["POST"],
"url": {
"path": "/_rank_eval",
"paths": ["/_rank_eval", "/{index}/_rank_eval", "/{index}/{type}/_rank_eval"],
"parts": {
"index": {
"type": "list",
"description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices"
},
"type": {
"type" : "list",
"description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types"
}
},
"params": {}
},
"body": {
"description": "The search definition using the Query DSL and the prototype for the eval request.",
"required": true
}
}
}

2
settings.gradle
View File

@ -37,6 +37,7 @@ List projects = [
'modules:mapper-extras',
'modules:parent-join',
'modules:percolator',
'modules:rank-eval',
'modules:reindex',
'modules:repository-url',
'modules:transport-netty4',
@ -77,6 +78,7 @@ List projects = [
'qa:smoke-test-ingest-with-all-dependencies',
'qa:smoke-test-ingest-disabled',
'qa:smoke-test-multinode',
'qa:smoke-test-rank-eval-with-mustache',
'qa:smoke-test-plugins',
'qa:smoke-test-reindex-with-all-modules',
'qa:smoke-test-tribe-node',