[Tests] Move templated _rank_eval tests (#30679)
This change moves the ranking evaluation tests that use templates to the existing yml rest tests instead of extending ESIntegTestCase. Closes #30628
This commit is contained in:
Christoph Büscher
GitHub
parent
2b8d3e8520
commit
9cb6b90a99
|
|
@ -31,6 +31,3 @@ dependencies {
|
||||||
* and will be fixed later.
|
* and will be fixed later.
|
||||||
* Tracked by https://github.com/elastic/elasticsearch/issues/30628
|
* Tracked by https://github.com/elastic/elasticsearch/issues/30628
|
||||||
*/
|
*/
|
||||||
if ("zip".equals(integTestCluster.distribution)) {
|
|
||||||
integTestRunner.enabled = false
|
|
||||||
}
|
|
||||||
|
|
|
||||||
|
|
@ -1,159 +0,0 @@
|
||||||
/*
|
|
||||||
* Licensed to Elasticsearch under one or more contributor
|
|
||||||
* license agreements. See the NOTICE file distributed with
|
|
||||||
* this work for additional information regarding copyright
|
|
||||||
* ownership. Elasticsearch licenses this file to you under
|
|
||||||
* the Apache License, Version 2.0 (the "License"); you may
|
|
||||||
* not use this file except in compliance with the License.
|
|
||||||
* You may obtain a copy of the License at
|
|
||||||
*
|
|
||||||
* http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
*
|
|
||||||
* Unless required by applicable law or agreed to in writing,
|
|
||||||
* software distributed under the License is distributed on an
|
|
||||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
||||||
* KIND, either express or implied. See the License for the
|
|
||||||
* specific language governing permissions and limitations
|
|
||||||
* under the License.
|
|
||||||
*/
|
|
||||||
|
|
||||||
package org.elasticsearch.index.rankeval;
|
|
||||||
|
|
||||||
import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId;
|
|
||||||
import org.elasticsearch.plugins.Plugin;
|
|
||||||
import org.elasticsearch.script.Script;
|
|
||||||
import org.elasticsearch.script.ScriptType;
|
|
||||||
import org.elasticsearch.test.ESIntegTestCase;
|
|
||||||
import org.junit.Before;
|
|
||||||
|
|
||||||
import java.io.IOException;
|
|
||||||
import java.util.ArrayList;
|
|
||||||
import java.util.Arrays;
|
|
||||||
import java.util.Collection;
|
|
||||||
import java.util.Collections;
|
|
||||||
import java.util.HashMap;
|
|
||||||
import java.util.HashSet;
|
|
||||||
import java.util.List;
|
|
||||||
import java.util.Map;
|
|
||||||
import java.util.Set;
|
|
||||||
|
|
||||||
|
|
||||||
public class SmokeMultipleTemplatesIT extends ESIntegTestCase {
|
|
||||||
|
|
||||||
private static final String MATCH_TEMPLATE = "match_template";
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
|
|
||||||
return Arrays.asList(RankEvalPlugin.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
protected Collection<Class<? extends Plugin>> nodePlugins() {
|
|
||||||
return Arrays.asList(RankEvalPlugin.class);
|
|
||||||
}
|
|
||||||
|
|
||||||
@Before
|
|
||||||
public void setup() {
|
|
||||||
createIndex("test");
|
|
||||||
ensureGreen();
|
|
||||||
|
|
||||||
client().prepareIndex("test", "testtype").setId("1")
|
|
||||||
.setSource("text", "berlin", "title", "Berlin, Germany").get();
|
|
||||||
client().prepareIndex("test", "testtype").setId("2")
|
|
||||||
.setSource("text", "amsterdam").get();
|
|
||||||
client().prepareIndex("test", "testtype").setId("3")
|
|
||||||
.setSource("text", "amsterdam").get();
|
|
||||||
client().prepareIndex("test", "testtype").setId("4")
|
|
||||||
.setSource("text", "amsterdam").get();
|
|
||||||
client().prepareIndex("test", "testtype").setId("5")
|
|
||||||
.setSource("text", "amsterdam").get();
|
|
||||||
client().prepareIndex("test", "testtype").setId("6")
|
|
||||||
.setSource("text", "amsterdam").get();
|
|
||||||
refresh();
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testPrecisionAtRequest() throws IOException {
|
|
||||||
|
|
||||||
List<RatedRequest> specifications = new ArrayList<>();
|
|
||||||
Map<String, Object> ams_params = new HashMap<>();
|
|
||||||
ams_params.put("querystring", "amsterdam");
|
|
||||||
RatedRequest amsterdamRequest = new RatedRequest(
|
|
||||||
"amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params, MATCH_TEMPLATE);
|
|
||||||
|
|
||||||
specifications.add(amsterdamRequest);
|
|
||||||
|
|
||||||
Map<String, Object> berlin_params = new HashMap<>();
|
|
||||||
berlin_params.put("querystring", "berlin");
|
|
||||||
RatedRequest berlinRequest = new RatedRequest(
|
|
||||||
"berlin_query", createRelevant("1"), berlin_params, MATCH_TEMPLATE);
|
|
||||||
specifications.add(berlinRequest);
|
|
||||||
|
|
||||||
PrecisionAtK metric = new PrecisionAtK();
|
|
||||||
|
|
||||||
ScriptWithId template =
|
|
||||||
new ScriptWithId(
|
|
||||||
MATCH_TEMPLATE,
|
|
||||||
new Script(
|
|
||||||
ScriptType.INLINE,
|
|
||||||
"mustache", "{\"query\": {\"match\": {\"text\": \"{{querystring}}\"}}}",
|
|
||||||
new HashMap<>()));
|
|
||||||
Set<ScriptWithId> templates = new HashSet<>();
|
|
||||||
templates.add(template);
|
|
||||||
RankEvalSpec task = new RankEvalSpec(specifications, metric, templates);
|
|
||||||
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
|
|
||||||
builder.setRankEvalSpec(task);
|
|
||||||
|
|
||||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices("test")).actionGet();
|
|
||||||
assertEquals(0.9, response.getEvaluationResult(), Double.MIN_VALUE);
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testTemplateWithAggsFails() {
|
|
||||||
String template = "{ \"aggs\" : { \"avg_grade\" : { \"avg\" : { \"field\" : \"grade\" }}}}";
|
|
||||||
assertTemplatedRequestFailures(template, "Query in rated requests should not contain aggregations.");
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testTemplateWithSuggestFails() {
|
|
||||||
String template = "{\"suggest\" : {\"my-suggestion\" : {\"text\" : \"Elastic\",\"term\" : {\"field\" : \"message\"}}}}";
|
|
||||||
assertTemplatedRequestFailures(template, "Query in rated requests should not contain a suggest section.");
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testTemplateWithHighlighterFails() {
|
|
||||||
String template = "{\"highlight\" : { \"fields\" : {\"content\" : {}}}}";
|
|
||||||
assertTemplatedRequestFailures(template, "Query in rated requests should not contain a highlighter section.");
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testTemplateWithProfileFails() {
|
|
||||||
String template = "{\"profile\" : \"true\" }";
|
|
||||||
assertTemplatedRequestFailures(template, "Query in rated requests should not use profile.");
|
|
||||||
}
|
|
||||||
|
|
||||||
public void testTemplateWithExplainFails() {
|
|
||||||
String template = "{\"explain\" : \"true\" }";
|
|
||||||
assertTemplatedRequestFailures(template, "Query in rated requests should not use explain.");
|
|
||||||
}
|
|
||||||
|
|
||||||
private static void assertTemplatedRequestFailures(String template, String expectedMessage) {
|
|
||||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
|
|
||||||
RatedRequest ratedRequest = new RatedRequest("id", ratedDocs, Collections.singletonMap("param1", "value1"), "templateId");
|
|
||||||
Collection<ScriptWithId> templates = Collections.singletonList(new ScriptWithId("templateId",
|
|
||||||
new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, template, Collections.emptyMap())));
|
|
||||||
RankEvalSpec rankEvalSpec = new RankEvalSpec(Collections.singletonList(ratedRequest), new PrecisionAtK(), templates);
|
|
||||||
RankEvalRequest rankEvalRequest = new RankEvalRequest(rankEvalSpec, new String[] { "test" });
|
|
||||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
|
||||||
() -> client().execute(RankEvalAction.INSTANCE, rankEvalRequest).actionGet());
|
|
||||||
assertEquals(expectedMessage, e.getMessage());
|
|
||||||
}
|
|
||||||
|
|
||||||
private static List<RatedDocument> createRelevant(String... docs) {
|
|
||||||
List<RatedDocument> relevant = new ArrayList<>();
|
|
||||||
for (String doc : docs) {
|
|
||||||
relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal()));
|
|
||||||
}
|
|
||||||
return relevant;
|
|
||||||
}
|
|
||||||
|
|
||||||
public enum Rating {
|
|
||||||
IRRELEVANT, RELEVANT;
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
@ -0,0 +1,171 @@
|
||||||
|
setup:
|
||||||
|
|
||||||
|
- do:
|
||||||
|
indices.create:
|
||||||
|
index: test
|
||||||
|
body:
|
||||||
|
settings:
|
||||||
|
index:
|
||||||
|
number_of_shards: 1
|
||||||
|
- do:
|
||||||
|
index:
|
||||||
|
index: test
|
||||||
|
type: _doc
|
||||||
|
id: 1
|
||||||
|
body: { "text": "berlin", "title" : "Berlin, Germany" }
|
||||||
|
|
||||||
|
- do:
|
||||||
|
index:
|
||||||
|
index: test
|
||||||
|
type: _doc
|
||||||
|
id: 2
|
||||||
|
body: { "text": "amsterdam" }
|
||||||
|
|
||||||
|
- do:
|
||||||
|
index:
|
||||||
|
index: test
|
||||||
|
type: _doc
|
||||||
|
id: 3
|
||||||
|
body: { "text": "amsterdam" }
|
||||||
|
|
||||||
|
- do:
|
||||||
|
index:
|
||||||
|
index: test
|
||||||
|
type: _doc
|
||||||
|
id: 4
|
||||||
|
body: { "text": "amsterdam" }
|
||||||
|
|
||||||
|
- do:
|
||||||
|
index:
|
||||||
|
index: test
|
||||||
|
type: _doc
|
||||||
|
id: 5
|
||||||
|
body: { "text": "amsterdam" }
|
||||||
|
|
||||||
|
- do:
|
||||||
|
index:
|
||||||
|
index: test
|
||||||
|
type: _doc
|
||||||
|
id: 6
|
||||||
|
body: { "text": "amsterdam" }
|
||||||
|
|
||||||
|
- do:
|
||||||
|
indices.refresh: {}
|
||||||
|
|
||||||
|
---
|
||||||
|
"Basic rank-eval request with template":
|
||||||
|
|
||||||
|
- skip:
|
||||||
|
version: " - 6.1.99"
|
||||||
|
reason: the ranking evaluation feature is available since 6.2
|
||||||
|
|
||||||
|
- do:
|
||||||
|
rank_eval:
|
||||||
|
body: {
|
||||||
|
"templates": [ { "id": "match", "template": {"source": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ],
|
||||||
|
"requests" : [
|
||||||
|
{
|
||||||
|
"id": "amsterdam_query",
|
||||||
|
"params": { "query_string": "amsterdam" },
|
||||||
|
"template_id": "match",
|
||||||
|
"ratings": [
|
||||||
|
{"_index": "test", "_id": "2", "rating": 1},
|
||||||
|
{"_index": "test", "_id": "3", "rating": 1},
|
||||||
|
{"_index": "test", "_id": "4", "rating": 1},
|
||||||
|
{"_index": "test", "_id": "5", "rating": 1},]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id" : "berlin_query",
|
||||||
|
"params": { "query_string": "berlin" },
|
||||||
|
"template_id": "match",
|
||||||
|
"ratings": [{"_index": "test", "_id": "1", "rating": 1}]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric" : { "precision": { }}
|
||||||
|
}
|
||||||
|
|
||||||
|
- match: {quality_level: 0.9}
|
||||||
|
- match: {details.amsterdam_query.unknown_docs.0._id: "6"}
|
||||||
|
|
||||||
|
---
|
||||||
|
"Test illegal request parts":
|
||||||
|
|
||||||
|
- do:
|
||||||
|
catch: /Query in rated requests should not contain aggregations./
|
||||||
|
rank_eval:
|
||||||
|
body: {
|
||||||
|
"templates": [ { "id": "match", "template": {"source": "{ \"aggs\" : { \"avg_grade\" : { \"avg\" : { \"field\" : \"grade\" }}}}" }} ],
|
||||||
|
"requests" : [
|
||||||
|
{
|
||||||
|
"id": "amsterdam_query",
|
||||||
|
"params": { "query_string": "amsterdam" },
|
||||||
|
"template_id": "match",
|
||||||
|
"ratings": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric" : { "precision": { }}
|
||||||
|
}
|
||||||
|
|
||||||
|
- do:
|
||||||
|
catch: /Query in rated requests should not contain a suggest section./
|
||||||
|
rank_eval:
|
||||||
|
body: {
|
||||||
|
"templates": [ { "id": "match", "template": {"source": "{\"suggest\" : {\"my-suggestion\" : {\"text\" : \"Elastic\",\"term\" : {\"field\" : \"message\"}}}}" }} ],
|
||||||
|
"requests" : [
|
||||||
|
{
|
||||||
|
"id": "amsterdam_query",
|
||||||
|
"params": { "query_string": "amsterdam" },
|
||||||
|
"template_id": "match",
|
||||||
|
"ratings": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric" : { "precision": { }}
|
||||||
|
}
|
||||||
|
|
||||||
|
- do:
|
||||||
|
catch: /Query in rated requests should not contain a highlighter section./
|
||||||
|
rank_eval:
|
||||||
|
body: {
|
||||||
|
"templates": [ { "id": "match", "template": {"source": "{\"highlight\" : { \"fields\" : {\"content\" : {}}}}" }} ],
|
||||||
|
"requests" : [
|
||||||
|
{
|
||||||
|
"id": "amsterdam_query",
|
||||||
|
"params": { "query_string": "amsterdam" },
|
||||||
|
"template_id": "match",
|
||||||
|
"ratings": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric" : { "precision": { }}
|
||||||
|
}
|
||||||
|
|
||||||
|
- do:
|
||||||
|
catch: /Query in rated requests should not use profile./
|
||||||
|
rank_eval:
|
||||||
|
body: {
|
||||||
|
"templates": [ { "id": "match", "template": {"source": "{\"profile\" : \"true\" }" }} ],
|
||||||
|
"requests" : [
|
||||||
|
{
|
||||||
|
"id": "amsterdam_query",
|
||||||
|
"params": { "query_string": "amsterdam" },
|
||||||
|
"template_id": "match",
|
||||||
|
"ratings": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric" : { "precision": { }}
|
||||||
|
}
|
||||||
|
|
||||||
|
- do:
|
||||||
|
catch: /Query in rated requests should not use explain./
|
||||||
|
rank_eval:
|
||||||
|
body: {
|
||||||
|
"templates": [ { "id": "match", "template": {"source": "{\"explain\" : \"true\" }" }} ],
|
||||||
|
"requests" : [
|
||||||
|
{
|
||||||
|
"id": "amsterdam_query",
|
||||||
|
"params": { "query_string": "amsterdam" },
|
||||||
|
"template_id": "match",
|
||||||
|
"ratings": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"metric" : { "precision": { }}
|
||||||
|
}
|
||||||
|
|
@ -1,72 +0,0 @@
|
||||||
---
|
|
||||||
"Template request":
|
|
||||||
|
|
||||||
- skip:
|
|
||||||
version: " - 6.1.99"
|
|
||||||
reason: the ranking evaluation feature is available since 6.2
|
|
||||||
|
|
||||||
- do:
|
|
||||||
indices.create:
|
|
||||||
index: foo
|
|
||||||
body:
|
|
||||||
settings:
|
|
||||||
index:
|
|
||||||
number_of_shards: 1
|
|
||||||
- do:
|
|
||||||
index:
|
|
||||||
index: foo
|
|
||||||
type: bar
|
|
||||||
id: doc1
|
|
||||||
body: { "text": "berlin" }
|
|
||||||
|
|
||||||
- do:
|
|
||||||
index:
|
|
||||||
index: foo
|
|
||||||
type: bar
|
|
||||||
id: doc2
|
|
||||||
body: { "text": "amsterdam" }
|
|
||||||
|
|
||||||
- do:
|
|
||||||
index:
|
|
||||||
index: foo
|
|
||||||
type: bar
|
|
||||||
id: doc3
|
|
||||||
body: { "text": "amsterdam" }
|
|
||||||
|
|
||||||
- do:
|
|
||||||
index:
|
|
||||||
index: foo
|
|
||||||
type: bar
|
|
||||||
id: doc4
|
|
||||||
body: { "text": "something about amsterdam and berlin" }
|
|
||||||
|
|
||||||
- do:
|
|
||||||
indices.refresh: {}
|
|
||||||
|
|
||||||
- do:
|
|
||||||
rank_eval:
|
|
||||||
body: {
|
|
||||||
"templates": [ { "id": "match", "template": {"source": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ],
|
|
||||||
"requests" : [
|
|
||||||
{
|
|
||||||
"id": "amsterdam_query",
|
|
||||||
"params": { "query_string": "amsterdam" },
|
|
||||||
"template_id": "match",
|
|
||||||
"ratings": [
|
|
||||||
{"_index": "foo", "_id": "doc1", "rating": 0},
|
|
||||||
{"_index": "foo", "_id": "doc2", "rating": 1},
|
|
||||||
{"_index": "foo", "_id": "doc3", "rating": 1}]
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"id" : "berlin_query",
|
|
||||||
"params": { "query_string": "berlin" },
|
|
||||||
"template_id": "match",
|
|
||||||
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
|
|
||||||
}
|
|
||||||
],
|
|
||||||
"metric" : { "precision": { }}
|
|
||||||
}
|
|
||||||
|
|
||||||
- match: {quality_level: 0.5833333333333333}
|
|
||||||
- match: {details.berlin_query.unknown_docs.0._id: "doc4"}
|
|
||||||
- match: {details.amsterdam_query.unknown_docs.0._id: "doc4"}
|
|
||||||
Loading…
Reference in New Issue