diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java index cfdbf966b05..da2f2ed9698 100644 --- a/core/src/main/java/org/elasticsearch/script/Script.java +++ b/core/src/main/java/org/elasticsearch/script/Script.java @@ -186,83 +186,84 @@ public final class Script implements ToXContent, Writeable { return builder; } - public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher) throws IOException { - return parse(parser, parseFieldMatcher, null); + public static Script parse(XContentParser parser, ParseFieldMatcher matcher) { + return parse(parser, matcher, null); } public static Script parse(XContentParser parser, QueryParseContext context) { + return parse(parser, context.getParseFieldMatcher(), null); + } + + public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, @Nullable String lang) { try { - return parse(parser, context.getParseFieldMatcher(), context.getDefaultScriptLanguage()); + XContentParser.Token token = parser.currentToken(); + // If the parser hasn't yet been pushed to the first token, do it now + if (token == null) { + token = parser.nextToken(); + } + if (token == XContentParser.Token.VALUE_STRING) { + return new Script(parser.text(), ScriptType.INLINE, lang, null); + } + if (token != XContentParser.Token.START_OBJECT) { + throw new ElasticsearchParseException("expected a string value or an object, but found [{}] instead", token); + } + String script = null; + ScriptType type = null; + Map params = null; + XContentType contentType = null; + String cfn = null; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + cfn = parser.currentName(); + } else if (parseFieldMatcher.match(cfn, ScriptType.INLINE.getParseField())) { + type = ScriptType.INLINE; + if (parser.currentToken() == XContentParser.Token.START_OBJECT) { + contentType = parser.contentType(); + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + script = builder.copyCurrentStructure(parser).bytes().utf8ToString(); + } else { + script = parser.text(); + } + } else if (parseFieldMatcher.match(cfn, ScriptType.FILE.getParseField())) { + type = ScriptType.FILE; + if (token == XContentParser.Token.VALUE_STRING) { + script = parser.text(); + } else { + throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); + } + } else if (parseFieldMatcher.match(cfn, ScriptType.STORED.getParseField())) { + type = ScriptType.STORED; + if (token == XContentParser.Token.VALUE_STRING) { + script = parser.text(); + } else { + throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); + } + } else if (parseFieldMatcher.match(cfn, ScriptField.LANG)) { + if (token == XContentParser.Token.VALUE_STRING) { + lang = parser.text(); + } else { + throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); + } + } else if (parseFieldMatcher.match(cfn, ScriptField.PARAMS)) { + if (token == XContentParser.Token.START_OBJECT) { + params = parser.map(); + } else { + throw new ElasticsearchParseException("expected an object for field [{}], but found [{}]", cfn, token); + } + } else { + throw new ElasticsearchParseException("unexpected field [{}]", cfn); + } + } + if (script == null) { + throw new ElasticsearchParseException("expected one of [{}], [{}] or [{}] fields, but found none", + ScriptType.INLINE.getParseField() .getPreferredName(), ScriptType.FILE.getParseField().getPreferredName(), + ScriptType.STORED.getParseField() .getPreferredName()); + } + return new Script(script, type, lang, params, contentType); } catch (IOException e) { throw new ParsingException(parser.getTokenLocation(), "Error parsing [" + ScriptField.SCRIPT.getPreferredName() + "] field", e); } - } - public static Script parse(XContentParser parser, ParseFieldMatcher parseFieldMatcher, @Nullable String lang) throws IOException { - XContentParser.Token token = parser.currentToken(); - // If the parser hasn't yet been pushed to the first token, do it now - if (token == null) { - token = parser.nextToken(); - } - if (token == XContentParser.Token.VALUE_STRING) { - return new Script(parser.text(), ScriptType.INLINE, lang, null); - } - if (token != XContentParser.Token.START_OBJECT) { - throw new ElasticsearchParseException("expected a string value or an object, but found [{}] instead", token); - } - String script = null; - ScriptType type = null; - Map params = null; - XContentType contentType = null; - String cfn = null; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - cfn = parser.currentName(); - } else if (parseFieldMatcher.match(cfn, ScriptType.INLINE.getParseField())) { - type = ScriptType.INLINE; - if (parser.currentToken() == XContentParser.Token.START_OBJECT) { - contentType = parser.contentType(); - XContentBuilder builder = XContentFactory.contentBuilder(contentType); - script = builder.copyCurrentStructure(parser).bytes().utf8ToString(); - } else { - script = parser.text(); - } - } else if (parseFieldMatcher.match(cfn, ScriptType.FILE.getParseField())) { - type = ScriptType.FILE; - if (token == XContentParser.Token.VALUE_STRING) { - script = parser.text(); - } else { - throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); - } - } else if (parseFieldMatcher.match(cfn, ScriptType.STORED.getParseField())) { - type = ScriptType.STORED; - if (token == XContentParser.Token.VALUE_STRING) { - script = parser.text(); - } else { - throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); - } - } else if (parseFieldMatcher.match(cfn, ScriptField.LANG)) { - if (token == XContentParser.Token.VALUE_STRING) { - lang = parser.text(); - } else { - throw new ElasticsearchParseException("expected a string value for field [{}], but found [{}]", cfn, token); - } - } else if (parseFieldMatcher.match(cfn, ScriptField.PARAMS)) { - if (token == XContentParser.Token.START_OBJECT) { - params = parser.map(); - } else { - throw new ElasticsearchParseException("expected an object for field [{}], but found [{}]", cfn, token); - } - } else { - throw new ElasticsearchParseException("unexpected field [{}]", cfn); - } - } - if (script == null) { - throw new ElasticsearchParseException("expected one of [{}], [{}] or [{}] fields, but found none", - ScriptType.INLINE.getParseField() .getPreferredName(), ScriptType.FILE.getParseField().getPreferredName(), - ScriptType.STORED.getParseField() .getPreferredName()); - } - return new Script(script, type, lang, params, contentType); } @Override @@ -287,7 +288,7 @@ public final class Script implements ToXContent, Writeable { @Override public String toString() { return "[script: " + script + ", type: " + type.getParseField().getPreferredName() + ", lang: " - + lang + ", params: " + params + "]"; + + lang + ", params: " + params + ", contentType: " + contentType + "]"; } public interface ScriptField { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 99d8b86a92f..d47610c52ad 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -51,7 +51,7 @@ public class PrecisionAtN implements RankedListQualityMetric { /** ratings equal or above this value will be considered relevant. */ private int relevantRatingThreshhold = 1; - public static final String NAME = "precisionatn"; + public static final String NAME = "precision_atn"; private static final ParseField SIZE_FIELD = new ParseField("size"); private static final ParseField RELEVANT_RATING_FIELD = new ParseField("relevant_rating_threshold"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java index 576fc594c9a..0e9a617bb59 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java @@ -23,6 +23,7 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.ParseFieldMatcherSupplier; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchExtRegistry; import org.elasticsearch.search.SearchRequestParsers; import org.elasticsearch.search.aggregations.AggregatorParsers; @@ -33,11 +34,14 @@ public class RankEvalContext implements ParseFieldMatcherSupplier { private final SearchRequestParsers searchRequestParsers; private final ParseFieldMatcher parseFieldMatcher; private final QueryParseContext parseContext; + private final ScriptService scriptService; - public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, SearchRequestParsers searchRequestParsers) { + public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, SearchRequestParsers searchRequestParsers, + ScriptService scriptService) { this.parseFieldMatcher = parseFieldMatcher; this.searchRequestParsers = searchRequestParsers; this.parseContext = parseContext; + this.scriptService = scriptService; } public Suggesters getSuggesters() { @@ -48,6 +52,14 @@ public class RankEvalContext implements ParseFieldMatcherSupplier { return searchRequestParsers.aggParsers; } + public SearchRequestParsers getSearchRequestParsers() { + return searchRequestParsers; + } + + public ScriptService getScriptService() { + return scriptService; + } + public SearchExtRegistry getSearchExtParsers() { return searchRequestParsers.searchExtParsers; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index cadb88d77c0..4e7f28ac901 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -20,19 +20,27 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Objects; +import java.util.Map; /** * This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric. @@ -42,11 +50,13 @@ import java.util.Objects; * */ public class RankEvalSpec extends ToXContentToBytes implements Writeable { - /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ private RankedListQualityMetric metric; + /** optional: Template to base test requests on */ + @Nullable + private Script template; public RankEvalSpec() { // TODO think if no args ctor is okay @@ -64,6 +74,9 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { ratedRequests.add(new RatedRequest(in)); } metric = in.readNamedWriteable(RankedListQualityMetric.class); + if (in.readBoolean()) { + template = new Script(in); + } } @Override @@ -73,22 +86,24 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { spec.writeTo(out); } out.writeNamedWriteable(metric); + if (template != null) { + out.writeBoolean(true); + template.writeTo(out); + } else { + out.writeBoolean(false); + } } - public void setEval(RankedListQualityMetric eval) { - this.metric = eval; + /** Set the metric to use for quality evaluation. */ + public void setMetric(RankedListQualityMetric metric) { + this.metric = metric; } - /** Returns the precision at n configuration (containing level of n to consider).*/ - public RankedListQualityMetric getEvaluator() { + /** Returns the metric to use for quality evaluation.*/ + public RankedListQualityMetric getMetric() { return metric; } - /** Sets the precision at n configuration (containing level of n to consider).*/ - public void setEvaluator(RankedListQualityMetric config) { - this.metric = config; - } - /** Returns a list of intent to query translation specifications to evaluate. */ public Collection getSpecifications() { return ratedRequests; @@ -98,19 +113,33 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { public void setSpecifications(Collection specifications) { this.ratedRequests = specifications; } + + /** Set the template to base test requests on. */ + public void setTemplate(Script script) { + this.template = script; + } + + /** Returns the template to base test requests on. */ + public Script getTemplate() { + return this.template; + } + private static final ParseField TEMPLATE_FIELD = new ParseField("template"); private static final ParseField METRIC_FIELD = new ParseField("metric"); private static final ParseField REQUESTS_FIELD = new ParseField("requests"); private static final ObjectParser PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new); static { - PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> { + PARSER.declareObject(RankEvalSpec::setMetric, (p, c) -> { try { return RankedListQualityMetric.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } } , METRIC_FIELD); + PARSER.declareObject(RankEvalSpec::setTemplate, (p, c) -> { + return Script.parse(p, c.getParseFieldMatcher(), "mustache"); + }, TEMPLATE_FIELD); PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { try { return RatedRequest.fromXContent(p, c); @@ -120,9 +149,43 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } , REQUESTS_FIELD); } + public static RankEvalSpec parse(XContentParser parser, RankEvalContext context, boolean templated) throws IOException { + RankEvalSpec spec = PARSER.parse(parser, context); + + if (templated) { + for (RatedRequest query_spec : spec.getSpecifications()) { + Map params = query_spec.getParams(); + Script scriptWithParams = new Script(spec.template.getScript(), spec.template.getType(), spec.template.getLang(), params); + String resolvedRequest = + ((BytesReference) + (context.getScriptService().executable(scriptWithParams, ScriptContext.Standard.SEARCH, params) + .run())) + .utf8ToString(); + try (XContentParser subParser = XContentFactory.xContent(resolvedRequest).createParser(resolvedRequest)) { + QueryParseContext parseContext = + new QueryParseContext( + context.getSearchRequestParsers().queryParsers, + subParser, + context.getParseFieldMatcher()); + SearchSourceBuilder templateResult = + SearchSourceBuilder.fromXContent( + parseContext, + context.getAggs(), + context.getSuggesters(), + context.getSearchExtParsers()); + query_spec.setTestRequest(templateResult); + } + } + } + return spec; + } + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); + if (this.template != null) { + builder.field(TEMPLATE_FIELD.getPreferredName(), this.template); + } builder.startArray(REQUESTS_FIELD.getPreferredName()); for (RatedRequest spec : this.ratedRequests) { spec.toXContent(builder, params); @@ -133,10 +196,6 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return builder; } - public static RankEvalSpec parse(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.parse(parser, context); - } - @Override public final boolean equals(Object obj) { if (this == obj) { @@ -146,12 +205,14 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return false; } RankEvalSpec other = (RankEvalSpec) obj; + return Objects.equals(ratedRequests, other.ratedRequests) && - Objects.equals(metric, other.metric); + Objects.equals(metric, other.metric) && + Objects.equals(template, other.template); } @Override public final int hashCode() { - return Objects.hash(ratedRequests, metric); + return Objects.hash(ratedRequests, metric, template); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index dbfc449b7ea..1cc3cc18a4a 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -32,17 +32,21 @@ import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Objects; +import java.util.Map; +import java.util.Map.Entry; + /** * Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request * template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well. * * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. * */ +@SuppressWarnings("unchecked") public class RatedRequest extends ToXContentToBytes implements Writeable { - private String specId; private SearchSourceBuilder testRequest; private List indices = new ArrayList<>(); @@ -50,6 +54,8 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private List summaryFields = new ArrayList<>(); /** Collection of rated queries for this query QA specification.*/ private List ratedDocs = new ArrayList<>(); + /** Map of parameters to use for filling a query template, can be used instead of providing testRequest. */ + private Map params = new HashMap<>(); public RatedRequest() { // ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser @@ -83,6 +89,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (int i = 0; i < intentSize; i++) { ratedDocs.add(new RatedDocument(in)); } + this.params = (Map) in.readMap(); int summaryFieldsSize = in.readInt(); summaryFields = new ArrayList<>(summaryFieldsSize); for (int i = 0; i < summaryFieldsSize; i++) { @@ -106,6 +113,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { for (RatedDocument ratedDoc : ratedDocs) { ratedDoc.writeTo(out); } + out.writeMap((Map) params); out.writeInt(summaryFields.size()); for (String fieldName : summaryFields) { out.writeString(fieldName); @@ -155,6 +163,14 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { public void setRatedDocs(List ratedDocs) { this.ratedDocs = ratedDocs; } + + public void setParams(Map params) { + this.params = params; + } + + public Map getParams() { + return this.params; + } public void setSummaryFields(List fields) { this.summaryFields = fields; @@ -168,6 +184,7 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { private static final ParseField ID_FIELD = new ParseField("id"); private static final ParseField REQUEST_FIELD = new ParseField("request"); private static final ParseField RATINGS_FIELD = new ParseField("ratings"); + private static final ParseField PARAMS_FIELD = new ParseField("params"); private static final ParseField FIELDS_FIELD = new ParseField("summary_fields"); private static final ObjectParser PARSER = new ObjectParser<>("requests", RatedRequest::new); @@ -187,7 +204,14 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); } - } , RATINGS_FIELD); + }, RATINGS_FIELD); + PARSER.declareObject(RatedRequest::setParams, (p, c) -> { + try { + return (Map) p.map(); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); + } + }, PARAMS_FIELD); } /** @@ -219,7 +243,13 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field(ID_FIELD.getPreferredName(), this.specId); - builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); + if (testRequest != null) + builder.field(REQUEST_FIELD.getPreferredName(), this.testRequest); + builder.startObject(PARAMS_FIELD.getPreferredName()); + for (Entry entry : this.params.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + builder.endObject(); builder.startArray(RATINGS_FIELD.getPreferredName()); for (RatedDocument doc : this.ratedDocs) { doc.toXContent(builder, params); @@ -250,11 +280,12 @@ public class RatedRequest extends ToXContentToBytes implements Writeable { Objects.equals(indices, other.indices) && Objects.equals(types, other.types) && Objects.equals(summaryFields, summaryFields) && - Objects.equals(ratedDocs, other.ratedDocs); + Objects.equals(ratedDocs, other.ratedDocs) && + Objects.equals(params, other.params); } @Override public final int hashCode() { - return Objects.hash(specId, testRequest, indices, types, summaryFields, ratedDocs); + return Objects.hash(specId, testRequest, indices, types, summaryFields, ratedDocs, params); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 3a629f94b45..ce06d794276 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -32,6 +32,7 @@ import org.elasticsearch.rest.RestController; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestActions; import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchRequestParsers; import java.io.IOException; @@ -156,19 +157,31 @@ import static org.elasticsearch.rest.RestRequest.Method.POST; * */ public class RestRankEvalAction extends BaseRestHandler { - private SearchRequestParsers searchRequestParsers; + private ScriptService scriptService; @Inject - public RestRankEvalAction(Settings settings, RestController controller, SearchRequestParsers searchRequestParsers) { + public RestRankEvalAction( + Settings settings, + RestController controller, + SearchRequestParsers searchRequestParsers, + ScriptService scriptService) { super(settings); this.searchRequestParsers = searchRequestParsers; + this.scriptService = scriptService; controller.registerHandler(GET, "/_rank_eval", this); controller.registerHandler(POST, "/_rank_eval", this); controller.registerHandler(GET, "/{index}/_rank_eval", this); controller.registerHandler(POST, "/{index}/_rank_eval", this); controller.registerHandler(GET, "/{index}/{type}/_rank_eval", this); controller.registerHandler(POST, "/{index}/{type}/_rank_eval", this); + + controller.registerHandler(GET, "/_rank_eval/template", this); + controller.registerHandler(POST, "/_rank_eval/template", this); + controller.registerHandler(GET, "/{index}/_rank_eval/template", this); + controller.registerHandler(POST, "/{index}/_rank_eval/template", this); + controller.registerHandler(GET, "/{index}/{type}/_rank_eval/template", this); + controller.registerHandler(POST, "/{index}/{type}/_rank_eval/template", this); } @Override @@ -180,7 +193,7 @@ public class RestRankEvalAction extends BaseRestHandler { if (restContent != null) { parseRankEvalRequest(rankEvalRequest, request, // TODO can we get rid of aggregators parsers and suggesters? - new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers)); + new RankEvalContext(parseFieldMatcher, parseContext, searchRequestParsers, scriptService)); } } return channel -> client.executeLocally(RankEvalAction.INSTANCE, rankEvalRequest, @@ -191,7 +204,9 @@ public class RestRankEvalAction extends BaseRestHandler { throws IOException { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); - RankEvalSpec spec = RankEvalSpec.parse(context.parser(), context); + RankEvalSpec spec = null; + boolean containsTemplate = request.path().contains("template"); + spec = RankEvalSpec.parse(context.parser(), context, containsTemplate); for (RatedRequest specification : spec.getSpecifications()) { specification.setIndices(indices); specification.setTypes(types); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 5225f12997e..d632534776f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -111,14 +111,14 @@ public class TransportRankEvalAction extends HandledTransportAction params = randomBoolean() ? null : Collections.singletonMap("key", "value"); + ScriptService.ScriptType scriptType = randomFrom(ScriptService.ScriptType.values()); + String script; + if (scriptType == ScriptService.ScriptType.INLINE) { + try (XContentBuilder builder = XContentBuilder.builder(xContent)) { + builder.startObject(); + builder.field("field", randomAsciiOfLengthBetween(1, 5)); + builder.endObject(); + script = builder.string(); + } + } else { + script = randomAsciiOfLengthBetween(1, 5); + } + + testItem.setTemplate(new Script( + script, + scriptType, + randomFrom("_lang1", "_lang2", null), + params, + scriptType == ScriptService.ScriptType.INLINE ? xContent.type() : null)); + } + + XContentBuilder builder = XContentFactory.contentBuilder(contentType); + if (ESTestCase.randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = ESTestCase.shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers); + searchRequestParsers, null); - RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext); + RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext, false); // IRL these come from URL parameters - see RestRankEvalAction parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); assertNotSame(testItem, parsedItem); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java index 01b749f759b..51d08b21c50 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedRequestsTests.java @@ -38,7 +38,9 @@ import org.junit.BeforeClass; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import static java.util.Collections.emptyList; @@ -81,6 +83,16 @@ public class RatedRequestsTests extends ESTestCase { RatedRequest ratedRequest = new RatedRequest(specId, testRequest, indices, types, ratedDocs); + + if (randomBoolean()) { + Map params = new HashMap(); + int randomSize = randomIntBetween(1, 10); + for (int i = 0; i < randomSize; i++) { + params.put(randomAsciiOfLengthBetween(1, 10), randomAsciiOfLengthBetween(1, 10)); + } + ratedRequest.setParams(params); + } + List summaryFields = new ArrayList<>(); int numSummaryFields = randomIntBetween(0, 5); for (int i = 0; i < numSummaryFields; i++) { @@ -109,7 +121,7 @@ public class RatedRequestsTests extends ESTestCase { QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers); + searchRequestParsers, null); RatedRequest parsedItem = RatedRequest.fromXContent(itemParser, rankContext); parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction @@ -143,7 +155,7 @@ public class RatedRequestsTests extends ESTestCase { XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, - searchRequestParsers); + searchRequestParsers, null); RatedRequest specification = RatedRequest.fromXContent(parser, rankContext); assertEquals("my_qa_query", specification.getSpecId()); assertNotNull(specification.getTestRequest()); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 36063a03a3f..da6646cb0a5 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -56,7 +56,7 @@ "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] } ], - "metric" : { "precisionatn": { "size": 10}} + "metric" : { "precision_atn": { "size": 10}} } - match: { rank_eval.quality_level: 1} diff --git a/qa/smoke-test-rank-eval-with-mustache/build.gradle b/qa/smoke-test-rank-eval-with-mustache/build.gradle new file mode 100644 index 00000000000..4860d5469af --- /dev/null +++ b/qa/smoke-test-rank-eval-with-mustache/build.gradle @@ -0,0 +1,27 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +apply plugin: 'elasticsearch.rest-test' + +/* +dependencies { + testCompile project(path: ':modules:rank-eval', configuration: 'runtime') + testCompile project(path: ':modules:lang-mustache', configuration: 'runtime') +} +*/ diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java new file mode 100644 index 00000000000..6fc2829c4a4 --- /dev/null +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/java/org/elasticsearch/smoketest/SmokeTestRankEvalWithMustacheYAMLTestSuiteIT.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.smoketest; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; + +import java.io.IOException; + +public class SmokeTestRankEvalWithMustacheYAMLTestSuiteIT extends ESClientYamlSuiteTestCase { + + public SmokeTestRankEvalWithMustacheYAMLTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, ClientYamlTestParseException { + return ESClientYamlSuiteTestCase.createParameters(0, 1); + } + +} diff --git a/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml new file mode 100644 index 00000000000..bf94ea6ae8f --- /dev/null +++ b/qa/smoke-test-rank-eval-with-mustache/src/test/resources/rest-api-spec/test/rank-eval/30_template.yaml @@ -0,0 +1,67 @@ +--- +"Template request": + - do: + indices.create: + index: foo + body: + settings: + index: + number_of_shards: 1 + - do: + index: + index: foo + type: bar + id: doc1 + body: { "text": "berlin" } + + - do: + index: + index: foo + type: bar + id: doc2 + body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: doc3 + body: { "text": "amsterdam" } + + - do: + index: + index: foo + type: bar + id: doc4 + body: { "text": "something about amsterdam and berlin" } + + - do: + indices.refresh: {} + + - do: + rank_eval_template: + body: { + "template": { + "inline": "{\"query\": { \"match\" : {\"text\" : \"{{query_string}}\" }}}" + }, + "requests" : [ + { + "id": "amsterdam_query", + "params": { "query_string": "amsterdam" }, + "ratings": [ + {"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 0}, + {"_index": "foo", "_type": "bar", "_id": "doc2", "rating": 1}, + {"_index": "foo", "_type": "bar", "_id": "doc3", "rating": 1}] + }, + { + "id" : "berlin_query", + "params": { "query_string": "berlin" }, + "ratings": [{"_index": "foo", "_type": "bar", "_id": "doc1", "rating": 1}] + } + ], + "metric" : { "precision_atn": { "size": 10}} + } + + - match: {rank_eval.quality_level: 1} + - match: {rank_eval.details.berlin_query.unknown_docs.0._id: "doc4"} + - match: {rank_eval.details.amsterdam_query.unknown_docs.0._id: "doc4"} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json index 681eb9f6081..37ff11f8764 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval.json @@ -4,9 +4,18 @@ "methods": ["POST"], "url": { "path": "/_rank_eval", - "paths": ["/_rank_eval"], - "parts": {}, - "params": {} + "paths": ["/_rank_eval", "/{index}/_rank_eval", "/{index}/{type}/_rank_eval"], + "parts": { + "index": { + "type": "list", + "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" + }, + "type": { + "type" : "list", + "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types" + } + }, + "params": {} }, "body": { "description": "The search definition using the Query DSL and the prototype for the eval request.", diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json new file mode 100644 index 00000000000..abb21c273e4 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/rank_eval_template.json @@ -0,0 +1,25 @@ +{ + "rank_eval_template": { + "documentation": "https://www.elastic.co/guide/en/elasticsearch/reference/master/docs-rank-eval.html", + "methods": ["POST"], + "url": { + "path": "/_rank_eval/template", + "paths": ["/_rank_eval/template", "/{index}/_rank_eval/template", "/{index}/{type}/_rank_eval/template"], + "parts": { + "index": { + "type": "list", + "description" : "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" + }, + "type": { + "type" : "list", + "description" : "A comma-separated list of document types to search; leave empty to perform the operation on all types" + } + }, + "params": {} + }, + "body": { + "description": "The search definition using the Query DSL and the prototype for the eval request.", + "required": true + } + } +} diff --git a/settings.gradle b/settings.gradle index aee42e1e923..4f249de4391 100644 --- a/settings.gradle +++ b/settings.gradle @@ -61,6 +61,7 @@ List projects = [ 'qa:smoke-test-ingest-with-all-dependencies', 'qa:smoke-test-ingest-disabled', 'qa:smoke-test-multinode', + 'qa:smoke-test-rank-eval-with-mustache', 'qa:smoke-test-plugins', 'qa:smoke-test-reindex-with-painless', 'qa:smoke-test-http',