Add tests for ranking evaluation with aliases (#29452)
The ranking evaluation requests so far were not tested against aliases but they should run regardless of the targeted index is a real index or an alias. This change adds cases for this to the integration and rest tests.
This commit is contained in:
Christoph Büscher
GitHub
parent
24763d881e
commit
d0f6657d90
|
|
@ -20,6 +20,7 @@
|
|||
package org.elasticsearch.index.rankeval;
|
||||
|
||||
import org.elasticsearch.ElasticsearchException;
|
||||
import org.elasticsearch.action.admin.indices.alias.IndicesAliasesRequest.AliasActions;
|
||||
import org.elasticsearch.action.search.SearchRequest;
|
||||
import org.elasticsearch.action.support.IndicesOptions;
|
||||
import org.elasticsearch.index.IndexNotFoundException;
|
||||
|
|
@ -40,10 +41,13 @@ import java.util.Map.Entry;
|
|||
import java.util.Set;
|
||||
|
||||
import static org.elasticsearch.index.rankeval.EvaluationMetric.filterUnknownDocuments;
|
||||
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
|
||||
import static org.hamcrest.Matchers.instanceOf;
|
||||
|
||||
public class RankEvalRequestIT extends ESIntegTestCase {
|
||||
|
||||
private static final String TEST_INDEX = "test";
|
||||
private static final String INDEX_ALIAS = "alias0";
|
||||
private static final int RELEVANT_RATING_1 = 1;
|
||||
|
||||
@Override
|
||||
|
|
@ -58,20 +62,23 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
|
||||
@Before
|
||||
public void setup() {
|
||||
createIndex("test");
|
||||
createIndex(TEST_INDEX);
|
||||
ensureGreen();
|
||||
|
||||
client().prepareIndex("test", "testtype").setId("1")
|
||||
client().prepareIndex(TEST_INDEX, "testtype").setId("1")
|
||||
.setSource("text", "berlin", "title", "Berlin, Germany", "population", 3670622).get();
|
||||
client().prepareIndex("test", "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex("test", "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex("test", "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex("test", "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex("test", "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex(TEST_INDEX, "testtype").setId("2").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex(TEST_INDEX, "testtype").setId("3").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex(TEST_INDEX, "testtype").setId("4").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex(TEST_INDEX, "testtype").setId("5").setSource("text", "amsterdam", "population", 851573).get();
|
||||
client().prepareIndex(TEST_INDEX, "testtype").setId("6").setSource("text", "amsterdam", "population", 851573).get();
|
||||
|
||||
// add another index for testing closed indices etc...
|
||||
client().prepareIndex("test2", "testtype").setId("7").setSource("text", "amsterdam", "population", 851573).get();
|
||||
refresh();
|
||||
|
||||
// set up an alias that can also be used in tests
|
||||
assertAcked(client().admin().indices().prepareAliases().addAliasAction(AliasActions.add().index(TEST_INDEX).alias(INDEX_ALIAS)));
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -101,7 +108,8 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
RankEvalAction.INSTANCE, new RankEvalRequest());
|
||||
builder.setRankEvalSpec(task);
|
||||
|
||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices("test"))
|
||||
String indexToUse = randomBoolean() ? TEST_INDEX : INDEX_ALIAS;
|
||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices(indexToUse))
|
||||
.actionGet();
|
||||
// the expected Prec@ for the first query is 4/6 and the expected Prec@ for the
|
||||
// second is 1/6, divided by 2 to get the average
|
||||
|
|
@ -143,7 +151,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
metric = new PrecisionAtK(1, false, 3);
|
||||
task = new RankEvalSpec(specifications, metric);
|
||||
|
||||
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
|
||||
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));
|
||||
|
||||
response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
|
||||
// if we look only at top 3 documente, the expected P@3 for the first query is
|
||||
|
|
@ -163,19 +171,19 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
|
||||
List<RatedRequest> specifications = new ArrayList<>();
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(
|
||||
new RatedDocument("test", "1", 3),
|
||||
new RatedDocument("test", "2", 2),
|
||||
new RatedDocument("test", "3", 3),
|
||||
new RatedDocument("test", "4", 0),
|
||||
new RatedDocument("test", "5", 1),
|
||||
new RatedDocument("test", "6", 2));
|
||||
new RatedDocument(TEST_INDEX, "1", 3),
|
||||
new RatedDocument(TEST_INDEX, "2", 2),
|
||||
new RatedDocument(TEST_INDEX, "3", 3),
|
||||
new RatedDocument(TEST_INDEX, "4", 0),
|
||||
new RatedDocument(TEST_INDEX, "5", 1),
|
||||
new RatedDocument(TEST_INDEX, "6", 2));
|
||||
specifications.add(new RatedRequest("amsterdam_query", ratedDocs, testQuery));
|
||||
|
||||
DiscountedCumulativeGain metric = new DiscountedCumulativeGain(false, null, 10);
|
||||
RankEvalSpec task = new RankEvalSpec(specifications, metric);
|
||||
|
||||
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
|
||||
new RankEvalRequest(task, new String[] { "test" }));
|
||||
new RankEvalRequest(task, new String[] { TEST_INDEX }));
|
||||
|
||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
|
||||
assertEquals(DiscountedCumulativeGainTests.EXPECTED_DCG, response.getEvaluationResult(), 10E-14);
|
||||
|
|
@ -184,7 +192,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
metric = new DiscountedCumulativeGain(false, null, 3);
|
||||
task = new RankEvalSpec(specifications, metric);
|
||||
|
||||
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
|
||||
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));
|
||||
|
||||
response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
|
||||
assertEquals(12.39278926071437, response.getEvaluationResult(), 10E-14);
|
||||
|
|
@ -203,7 +211,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
RankEvalSpec task = new RankEvalSpec(specifications, metric);
|
||||
|
||||
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
|
||||
new RankEvalRequest(task, new String[] { "test" }));
|
||||
new RankEvalRequest(task, new String[] { TEST_INDEX }));
|
||||
|
||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
|
||||
// the expected reciprocal rank for the amsterdam_query is 1/5
|
||||
|
|
@ -216,7 +224,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
metric = new MeanReciprocalRank(1, 3);
|
||||
task = new RankEvalSpec(specifications, metric);
|
||||
|
||||
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { "test" }));
|
||||
builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest(task, new String[] { TEST_INDEX }));
|
||||
|
||||
response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
|
||||
// limiting to top 3 results, the amsterdam_query has no relevant document in it
|
||||
|
|
@ -247,7 +255,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
|
||||
|
||||
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE,
|
||||
new RankEvalRequest(task, new String[] { "test" }));
|
||||
new RankEvalRequest(task, new String[] { TEST_INDEX }));
|
||||
builder.setRankEvalSpec(task);
|
||||
|
||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
|
||||
|
|
@ -267,7 +275,7 @@ public class RankEvalRequestIT extends ESIntegTestCase {
|
|||
specifications.add(new RatedRequest("amsterdam_query", relevantDocs, amsterdamQuery));
|
||||
RankEvalSpec task = new RankEvalSpec(specifications, new PrecisionAtK());
|
||||
|
||||
RankEvalRequest request = new RankEvalRequest(task, new String[] { "test", "test2" });
|
||||
RankEvalRequest request = new RankEvalRequest(task, new String[] { TEST_INDEX, "test2" });
|
||||
request.setRankEvalSpec(task);
|
||||
|
||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, request).actionGet();
|
||||
|
|
|
|||
|
|
@ -1,10 +1,4 @@
|
|||
---
|
||||
"Response format":
|
||||
|
||||
- skip:
|
||||
version: " - 6.2.99"
|
||||
reason: response format was updated in 6.3
|
||||
|
||||
setup:
|
||||
- do:
|
||||
indices.create:
|
||||
index: foo
|
||||
|
|
@ -43,8 +37,21 @@
|
|||
- do:
|
||||
indices.refresh: {}
|
||||
|
||||
- do:
|
||||
indices.put_alias:
|
||||
index: foo
|
||||
name: alias
|
||||
|
||||
---
|
||||
"Response format":
|
||||
|
||||
- skip:
|
||||
version: " - 6.2.99"
|
||||
reason: response format was updated in 6.3
|
||||
|
||||
- do:
|
||||
rank_eval:
|
||||
index: foo,
|
||||
body: {
|
||||
"requests" : [
|
||||
{
|
||||
|
|
@ -86,6 +93,35 @@
|
|||
- match: { details.berlin_query.hits.1.hit._id: "doc4" }
|
||||
- is_false: details.berlin_query.hits.1.rating
|
||||
|
||||
---
|
||||
"Alias resolution":
|
||||
|
||||
- do:
|
||||
rank_eval:
|
||||
index: alias
|
||||
body: {
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"request": { "query": { "match" : {"text" : "amsterdam" }}},
|
||||
"ratings": [
|
||||
{"_index": "foo", "_id": "doc1", "rating": 0},
|
||||
{"_index": "foo", "_id": "doc2", "rating": 1},
|
||||
{"_index": "foo", "_id": "doc3", "rating": 1}]
|
||||
},
|
||||
{
|
||||
"id" : "berlin_query",
|
||||
"request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
|
||||
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { "ignore_unlabeled" : true }}
|
||||
}
|
||||
|
||||
- match: { quality_level: 1}
|
||||
- match: { details.amsterdam_query.quality_level: 1.0}
|
||||
- match: { details.berlin_query.quality_level: 1.0}
|
||||
|
||||
---
|
||||
"Mean Reciprocal Rank":
|
||||
|
||||
|
|
@ -93,44 +129,6 @@
|
|||
version: " - 6.2.99"
|
||||
reason: response format was updated in 6.3
|
||||
|
||||
- do:
|
||||
indices.create:
|
||||
index: foo
|
||||
body:
|
||||
settings:
|
||||
index:
|
||||
number_of_shards: 1
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc1
|
||||
body: { "text": "berlin" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc2
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc3
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc4
|
||||
body: { "text": "something about amsterdam and berlin" }
|
||||
|
||||
- do:
|
||||
indices.refresh: {}
|
||||
|
||||
- do:
|
||||
rank_eval:
|
||||
body: {
|
||||
|
|
|
|||
Loading…
Reference in New Issue