[Tests] Move templated _rank_eval tests (#30679)
This change moves the ranking evaluation tests that use templates to the existing yml rest tests instead of extending ESIntegTestCase. Closes #30628
This commit is contained in:
Christoph Büscher
GitHub
parent
2b8d3e8520
commit
9cb6b90a99
|
|
@ -31,6 +31,3 @@ dependencies {
|
|||
* and will be fixed later.
|
||||
* Tracked by https://github.com/elastic/elasticsearch/issues/30628
|
||||
*/
|
||||
if ("zip".equals(integTestCluster.distribution)) {
|
||||
integTestRunner.enabled = false
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,159 +0,0 @@
|
|||
/*
|
||||
* Licensed to Elasticsearch under one or more contributor
|
||||
* license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright
|
||||
* ownership. Elasticsearch licenses this file to you under
|
||||
* the Apache License, Version 2.0 (the "License"); you may
|
||||
* not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.elasticsearch.index.rankeval;
|
||||
|
||||
import org.elasticsearch.index.rankeval.RankEvalSpec.ScriptWithId;
|
||||
import org.elasticsearch.plugins.Plugin;
|
||||
import org.elasticsearch.script.Script;
|
||||
import org.elasticsearch.script.ScriptType;
|
||||
import org.elasticsearch.test.ESIntegTestCase;
|
||||
import org.junit.Before;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collection;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
|
||||
|
||||
public class SmokeMultipleTemplatesIT extends ESIntegTestCase {
|
||||
|
||||
private static final String MATCH_TEMPLATE = "match_template";
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
|
||||
return Arrays.asList(RankEvalPlugin.class);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Collection<Class<? extends Plugin>> nodePlugins() {
|
||||
return Arrays.asList(RankEvalPlugin.class);
|
||||
}
|
||||
|
||||
@Before
|
||||
public void setup() {
|
||||
createIndex("test");
|
||||
ensureGreen();
|
||||
|
||||
client().prepareIndex("test", "testtype").setId("1")
|
||||
.setSource("text", "berlin", "title", "Berlin, Germany").get();
|
||||
client().prepareIndex("test", "testtype").setId("2")
|
||||
.setSource("text", "amsterdam").get();
|
||||
client().prepareIndex("test", "testtype").setId("3")
|
||||
.setSource("text", "amsterdam").get();
|
||||
client().prepareIndex("test", "testtype").setId("4")
|
||||
.setSource("text", "amsterdam").get();
|
||||
client().prepareIndex("test", "testtype").setId("5")
|
||||
.setSource("text", "amsterdam").get();
|
||||
client().prepareIndex("test", "testtype").setId("6")
|
||||
.setSource("text", "amsterdam").get();
|
||||
refresh();
|
||||
}
|
||||
|
||||
public void testPrecisionAtRequest() throws IOException {
|
||||
|
||||
List<RatedRequest> specifications = new ArrayList<>();
|
||||
Map<String, Object> ams_params = new HashMap<>();
|
||||
ams_params.put("querystring", "amsterdam");
|
||||
RatedRequest amsterdamRequest = new RatedRequest(
|
||||
"amsterdam_query", createRelevant("2", "3", "4", "5"), ams_params, MATCH_TEMPLATE);
|
||||
|
||||
specifications.add(amsterdamRequest);
|
||||
|
||||
Map<String, Object> berlin_params = new HashMap<>();
|
||||
berlin_params.put("querystring", "berlin");
|
||||
RatedRequest berlinRequest = new RatedRequest(
|
||||
"berlin_query", createRelevant("1"), berlin_params, MATCH_TEMPLATE);
|
||||
specifications.add(berlinRequest);
|
||||
|
||||
PrecisionAtK metric = new PrecisionAtK();
|
||||
|
||||
ScriptWithId template =
|
||||
new ScriptWithId(
|
||||
MATCH_TEMPLATE,
|
||||
new Script(
|
||||
ScriptType.INLINE,
|
||||
"mustache", "{\"query\": {\"match\": {\"text\": \"{{querystring}}\"}}}",
|
||||
new HashMap<>()));
|
||||
Set<ScriptWithId> templates = new HashSet<>();
|
||||
templates.add(template);
|
||||
RankEvalSpec task = new RankEvalSpec(specifications, metric, templates);
|
||||
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(client(), RankEvalAction.INSTANCE, new RankEvalRequest());
|
||||
builder.setRankEvalSpec(task);
|
||||
|
||||
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request().indices("test")).actionGet();
|
||||
assertEquals(0.9, response.getEvaluationResult(), Double.MIN_VALUE);
|
||||
}
|
||||
|
||||
public void testTemplateWithAggsFails() {
|
||||
String template = "{ \"aggs\" : { \"avg_grade\" : { \"avg\" : { \"field\" : \"grade\" }}}}";
|
||||
assertTemplatedRequestFailures(template, "Query in rated requests should not contain aggregations.");
|
||||
}
|
||||
|
||||
public void testTemplateWithSuggestFails() {
|
||||
String template = "{\"suggest\" : {\"my-suggestion\" : {\"text\" : \"Elastic\",\"term\" : {\"field\" : \"message\"}}}}";
|
||||
assertTemplatedRequestFailures(template, "Query in rated requests should not contain a suggest section.");
|
||||
}
|
||||
|
||||
public void testTemplateWithHighlighterFails() {
|
||||
String template = "{\"highlight\" : { \"fields\" : {\"content\" : {}}}}";
|
||||
assertTemplatedRequestFailures(template, "Query in rated requests should not contain a highlighter section.");
|
||||
}
|
||||
|
||||
public void testTemplateWithProfileFails() {
|
||||
String template = "{\"profile\" : \"true\" }";
|
||||
assertTemplatedRequestFailures(template, "Query in rated requests should not use profile.");
|
||||
}
|
||||
|
||||
public void testTemplateWithExplainFails() {
|
||||
String template = "{\"explain\" : \"true\" }";
|
||||
assertTemplatedRequestFailures(template, "Query in rated requests should not use explain.");
|
||||
}
|
||||
|
||||
private static void assertTemplatedRequestFailures(String template, String expectedMessage) {
|
||||
List<RatedDocument> ratedDocs = Arrays.asList(new RatedDocument("index1", "id1", 1));
|
||||
RatedRequest ratedRequest = new RatedRequest("id", ratedDocs, Collections.singletonMap("param1", "value1"), "templateId");
|
||||
Collection<ScriptWithId> templates = Collections.singletonList(new ScriptWithId("templateId",
|
||||
new Script(ScriptType.INLINE, Script.DEFAULT_TEMPLATE_LANG, template, Collections.emptyMap())));
|
||||
RankEvalSpec rankEvalSpec = new RankEvalSpec(Collections.singletonList(ratedRequest), new PrecisionAtK(), templates);
|
||||
RankEvalRequest rankEvalRequest = new RankEvalRequest(rankEvalSpec, new String[] { "test" });
|
||||
IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
|
||||
() -> client().execute(RankEvalAction.INSTANCE, rankEvalRequest).actionGet());
|
||||
assertEquals(expectedMessage, e.getMessage());
|
||||
}
|
||||
|
||||
private static List<RatedDocument> createRelevant(String... docs) {
|
||||
List<RatedDocument> relevant = new ArrayList<>();
|
||||
for (String doc : docs) {
|
||||
relevant.add(new RatedDocument("test", doc, Rating.RELEVANT.ordinal()));
|
||||
}
|
||||
return relevant;
|
||||
}
|
||||
|
||||
public enum Rating {
|
||||
IRRELEVANT, RELEVANT;
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,171 @@
|
|||
setup:
|
||||
|
||||
- do:
|
||||
indices.create:
|
||||
index: test
|
||||
body:
|
||||
settings:
|
||||
index:
|
||||
number_of_shards: 1
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: _doc
|
||||
id: 1
|
||||
body: { "text": "berlin", "title" : "Berlin, Germany" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: _doc
|
||||
id: 2
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: _doc
|
||||
id: 3
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: _doc
|
||||
id: 4
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: _doc
|
||||
id: 5
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: test
|
||||
type: _doc
|
||||
id: 6
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
indices.refresh: {}
|
||||
|
||||
---
|
||||
"Basic rank-eval request with template":
|
||||
|
||||
- skip:
|
||||
version: " - 6.1.99"
|
||||
reason: the ranking evaluation feature is available since 6.2
|
||||
|
||||
- do:
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"source": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": [
|
||||
{"_index": "test", "_id": "2", "rating": 1},
|
||||
{"_index": "test", "_id": "3", "rating": 1},
|
||||
{"_index": "test", "_id": "4", "rating": 1},
|
||||
{"_index": "test", "_id": "5", "rating": 1},]
|
||||
},
|
||||
{
|
||||
"id" : "berlin_query",
|
||||
"params": { "query_string": "berlin" },
|
||||
"template_id": "match",
|
||||
"ratings": [{"_index": "test", "_id": "1", "rating": 1}]
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
}
|
||||
|
||||
- match: {quality_level: 0.9}
|
||||
- match: {details.amsterdam_query.unknown_docs.0._id: "6"}
|
||||
|
||||
---
|
||||
"Test illegal request parts":
|
||||
|
||||
- do:
|
||||
catch: /Query in rated requests should not contain aggregations./
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"source": "{ \"aggs\" : { \"avg_grade\" : { \"avg\" : { \"field\" : \"grade\" }}}}" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": []
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
}
|
||||
|
||||
- do:
|
||||
catch: /Query in rated requests should not contain a suggest section./
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"source": "{\"suggest\" : {\"my-suggestion\" : {\"text\" : \"Elastic\",\"term\" : {\"field\" : \"message\"}}}}" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": []
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
}
|
||||
|
||||
- do:
|
||||
catch: /Query in rated requests should not contain a highlighter section./
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"source": "{\"highlight\" : { \"fields\" : {\"content\" : {}}}}" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": []
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
}
|
||||
|
||||
- do:
|
||||
catch: /Query in rated requests should not use profile./
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"source": "{\"profile\" : \"true\" }" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": []
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
}
|
||||
|
||||
- do:
|
||||
catch: /Query in rated requests should not use explain./
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"source": "{\"explain\" : \"true\" }" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": []
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
}
|
||||
|
|
@ -1,72 +0,0 @@
|
|||
---
|
||||
"Template request":
|
||||
|
||||
- skip:
|
||||
version: " - 6.1.99"
|
||||
reason: the ranking evaluation feature is available since 6.2
|
||||
|
||||
- do:
|
||||
indices.create:
|
||||
index: foo
|
||||
body:
|
||||
settings:
|
||||
index:
|
||||
number_of_shards: 1
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc1
|
||||
body: { "text": "berlin" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc2
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc3
|
||||
body: { "text": "amsterdam" }
|
||||
|
||||
- do:
|
||||
index:
|
||||
index: foo
|
||||
type: bar
|
||||
id: doc4
|
||||
body: { "text": "something about amsterdam and berlin" }
|
||||
|
||||
- do:
|
||||
indices.refresh: {}
|
||||
|
||||
- do:
|
||||
rank_eval:
|
||||
body: {
|
||||
"templates": [ { "id": "match", "template": {"source": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" }} ],
|
||||
"requests" : [
|
||||
{
|
||||
"id": "amsterdam_query",
|
||||
"params": { "query_string": "amsterdam" },
|
||||
"template_id": "match",
|
||||
"ratings": [
|
||||
{"_index": "foo", "_id": "doc1", "rating": 0},
|
||||
{"_index": "foo", "_id": "doc2", "rating": 1},
|
||||
{"_index": "foo", "_id": "doc3", "rating": 1}]
|
||||
},
|
||||
{
|
||||
"id" : "berlin_query",
|
||||
"params": { "query_string": "berlin" },
|
||||
"template_id": "match",
|
||||
"ratings": [{"_index": "foo", "_id": "doc1", "rating": 1}]
|
||||
}
|
||||
],
|
||||
"metric" : { "precision": { }}
|
||||
}
|
||||
|
||||
- match: {quality_level: 0.5833333333333333}
|
||||
- match: {details.berlin_query.unknown_docs.0._id: "doc4"}
|
||||
- match: {details.amsterdam_query.unknown_docs.0._id: "doc4"}
|
||||
Loading…
Reference in New Issue