honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #19283 from cbuescher/rank-add-restparsing 

Query evaluation: Adding rest layer parsing and response rendering
This commit is contained in:
Isabel Drost-Fromm 2016-07-21 13:11:31 +02:00 committed by GitHub
parent 00a11b77fb b730494bfc
commit 2b3abadad1
19 changed files with 810 additions and 476 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java
View File

@ -21,25 +21,25 @@ package org.elasticsearch.index.rankeval;
import java.util.Collection;
/** Returned for each search intent and search specification combination. Summarises the document ids found that were not
* annotated and the average precision of result sets in each particular combination based on the annotations given.
/** Returned for each search specification. Summarizes the measured quality metric for this search request
* and adds the document ids found that were in the search result but not annotated in the original request.
* */
public class EvalQueryQuality {
private double qualityLevel;
private Collection<String> unknownDocs;
public EvalQueryQuality (double qualityLevel, Collection<String> unknownDocs) {
this.qualityLevel = qualityLevel;
this.unknownDocs = unknownDocs;
}
public Collection<String> getUnknownDocs() {
return unknownDocs;
}
public double getQualityLevel() {
return qualityLevel;
return qualityLevel;
}
}

28
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/Evaluator.java
View File

@ -1,28 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.search.SearchHit;
public interface Evaluator extends NamedWriteable {
Object evaluate(SearchHit[] hits, RatedQuery intent);
}

66
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
View File

@ -19,28 +19,31 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParseFieldMatcherSupplier;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ConstructingObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.SearchHit;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
import java.util.Map.Entry;
import java.util.List;
import javax.naming.directory.SearchResult;
/**
* Evaluate Precision at N, N being the number of search results to consider for precision calculation.
*
*
* Documents of unkonwn quality are ignored in the precision at n computation and returned by document id.
* */
public class PrecisionAtN implements RankedListQualityMetric {
public class PrecisionAtN extends RankedListQualityMetric {
/** Number of results to check against a given set of relevant results. */
private int n;
public static final String NAME = "precisionatn";
public PrecisionAtN(StreamInput in) throws IOException {
@ -63,7 +66,7 @@ public class PrecisionAtN implements RankedListQualityMetric {
public PrecisionAtN() {
this.n = 10;
}
/**
* @param n number of top results to check against a given set of relevant results.
* */
@ -78,24 +81,31 @@ public class PrecisionAtN implements RankedListQualityMetric {
return n;
}
private static final ParseField SIZE_FIELD = new ParseField("size");
private static final ConstructingObjectParser<PrecisionAtN, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>(
"precision_at", a -> new PrecisionAtN((Integer) a[0]));
static {
PARSER.declareInt(ConstructingObjectParser.constructorArg(), SIZE_FIELD);
}
public static PrecisionAtN fromXContent(XContentParser parser, ParseFieldMatcherSupplier matcher) {
return PARSER.apply(parser, matcher);
}
/** Compute precisionAtN based on provided relevant document IDs.
* @return precision at n for above {@link SearchResult} list.
**/
@Override
public EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent) {
Map<String, Integer> ratedDocIds = intent.getRatedDocuments();
Collection<String> relevantDocIds = new ArrayList<>();
for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
if (Rating.RELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
relevantDocIds.add(entry.getKey());
}
}
Collection<String> irrelevantDocIds = new ArrayList<>();
for (Entry<String, Integer> entry : ratedDocIds.entrySet()) {
if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(entry.getValue()))) {
irrelevantDocIds.add(entry.getKey());
public EvalQueryQuality evaluate(SearchHit[] hits, List<RatedDocument> ratedDocs) {
Collection<String> relevantDocIds = new ArrayList<>();
Collection<String> irrelevantDocIds = new ArrayList<>();
for (RatedDocument doc : ratedDocs) {
if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
relevantDocIds.add(doc.getDocID());
} else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) {
irrelevantDocIds.add(doc.getDocID());
}
}
@ -117,24 +127,24 @@ public class PrecisionAtN implements RankedListQualityMetric {
return new EvalQueryQuality(precision, unknownDocIds);
}
public enum Rating {
RELEVANT, IRRELEVANT;
IRRELEVANT, RELEVANT;
}
/**
* Needed to get the enum accross serialisation boundaries.
* */
public static class RatingMapping {
public static Integer mapFrom(Rating rating) {
if (Rating.RELEVANT.equals(rating)) {
return 0;
return 1;
}
return 1;
return 0;
}
public static Rating mapTo(Integer rating) {
if (rating == 0) {
if (rating == 1) {
return Rating.RELEVANT;
}
return Rating.IRRELEVANT;

96
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java
View File

@ -19,9 +19,13 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import java.io.IOException;
@ -32,25 +36,33 @@ import java.util.List;
* Defines a QA specification: All end user supplied query intents will be mapped to the search request specified in this search request
* template and executed against the targetIndex given. Any filters that should be applied in the target system can be specified as well.
*
* The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery.
* The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery.
* */
public class QuerySpec implements Writeable {
private int specId = 0;
private String specId;
private SearchSourceBuilder testRequest;
private List<String> indices = new ArrayList<>();
private List<String> types = new ArrayList<>();
public QuerySpec(
int specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types) {
/** Collection of rated queries for this query QA specification.*/
private List<RatedDocument> ratedDocs = new ArrayList<>();
public QuerySpec() {
// ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser
// TODO decide if we can require only id as mandatory, set default values for the rest?
}
public QuerySpec(String specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types,
List<RatedDocument> ratedDocs) {
this.specId = specId;
this.testRequest = testRequest;
this.indices = indices;
this.types = types;
this.ratedDocs = ratedDocs;
}
public QuerySpec(StreamInput in) throws IOException {
this.specId = in.readInt();
this.specId = in.readString();
testRequest = new SearchSourceBuilder(in);
int indicesSize = in.readInt();
indices = new ArrayList<String>(indicesSize);
@ -62,11 +74,16 @@ public class QuerySpec implements Writeable {
for (int i = 0; i < typesSize; i++) {
this.types.add(in.readString());
}
int intentSize = in.readInt();
ratedDocs = new ArrayList<>(intentSize);
for (int i = 0; i < intentSize; i++) {
ratedDocs.add(new RatedDocument(in));
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(specId);
out.writeString(specId);
testRequest.writeTo(out);
out.writeInt(indices.size());
for (String index : indices) {
@ -76,6 +93,10 @@ public class QuerySpec implements Writeable {
for (String type : types) {
out.writeString(type);
}
out.writeInt(ratedDocs.size());
for (RatedDocument ratedDoc : ratedDocs) {
ratedDoc.writeTo(out);
}
}
public SearchSourceBuilder getTestRequest() {
@ -103,12 +124,69 @@ public class QuerySpec implements Writeable {
}
/** Returns a user supplied spec id for easier referencing. */
public int getSpecId() {
public String getSpecId() {
return specId;
}
/** Sets a user supplied spec id for easier referencing. */
public void setSpecId(int specId) {
public void setSpecId(String specId) {
this.specId = specId;
}
/** Returns a list of rated documents to evaluate. */
public List<RatedDocument> getRatedDocs() {
return ratedDocs;
}
/** Set a list of rated documents for this query. */
public void setRatedDocs(List<RatedDocument> ratedDocs) {
this.ratedDocs = ratedDocs;
}
private static final ParseField ID_FIELD = new ParseField("id");
private static final ParseField REQUEST_FIELD = new ParseField("request");
private static final ParseField RATINGS_FIELD = new ParseField("ratings");
private static final ObjectParser<QuerySpec, RankEvalContext> PARSER = new ObjectParser<>("requests", QuerySpec::new);
static {
PARSER.declareString(QuerySpec::setSpecId, ID_FIELD);
PARSER.declareObject(QuerySpec::setTestRequest, (p, c) -> {
try {
return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters());
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing request", ex);
}
} , REQUEST_FIELD);
PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> {
try {
return RatedDocument.fromXContent(p);
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex);
}
} , RATINGS_FIELD);
}
/**
* Parses {@link QuerySpec} from rest representation:
*
* Example:
* {
* "id": "coffee_query",
* "request": {
* "query": {
* "bool": {
* "must": [
* {"match": {"beverage": "coffee"}},
* {"term": {"browser": {"value": "safari"}}},
* {"term": {"time_of_day": {"value": "morning","boost": 2}}},
* {"term": {"ip_location": {"value": "ams","boost": 10}}}]}
* },
* "size": 10
* },
* "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ]
* }
*/
public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException {
return PARSER.parse(parser, context);
}
}

65
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalContext.java Normal file
View File

@ -0,0 +1,65 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParseFieldMatcherSupplier;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.search.aggregations.AggregatorParsers;
import org.elasticsearch.search.suggest.Suggesters;
public class RankEvalContext implements ParseFieldMatcherSupplier {
private final ParseFieldMatcher parseFieldMatcher;
private final AggregatorParsers aggs;
private final Suggesters suggesters;
private final QueryParseContext parseContext;
public RankEvalContext(ParseFieldMatcher parseFieldMatcher, QueryParseContext parseContext, AggregatorParsers aggs,
Suggesters suggesters) {
this.parseFieldMatcher = parseFieldMatcher;
this.aggs = aggs;
this.suggesters = suggesters;
this.parseContext = parseContext;
}
public Suggesters getSuggesters() {
return this.suggesters;
}
public AggregatorParsers getAggs() {
return this.aggs;
}
@Override
public ParseFieldMatcher getParseFieldMatcher() {
return this.parseFieldMatcher;
}
public XContentParser parser() {
return this.parseContext.parser();
}
public QueryParseContext getParseContext() {
return this.parseContext;
}
}

16
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalRequest.java
View File

@ -35,20 +35,24 @@ public class RankEvalRequest extends ActionRequest<RankEvalRequest> {
/** The request data to use for evaluation. */
private RankEvalSpec task;
@Override
public ActionRequestValidationException validate() {
return null; // TODO
return null; // TODO
}
/** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics
* to compute. */
/**
* Returns the specification of this qa run including intents to execute,
* specifications detailing intent translation and metrics to compute.
*/
public RankEvalSpec getRankEvalSpec() {
return task;
}
/** Returns the specification of this qa run including intents to execute, specifications detailing intent translation and metrics
* to compute. */
/**
* Returns the specification of this qa run including intents to execute,
* specifications detailing intent translation and metrics to compute.
*/
public void setRankEvalSpec(RankEvalSpec task) {
this.task = task;
}

63
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java
View File

@ -22,54 +22,65 @@ package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.ActionResponse;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Map;
/**
/**
* For each qa specification identified by its id this response returns the respective
* averaged precisionAnN value.
*
*
* In addition for each query the document ids that haven't been found annotated is returned as well.
*
*
* Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned
* by the search are not taken into consideration when computing precision at n - they are ignored.
*
*
**/
public class RankEvalResponse extends ActionResponse {
public class RankEvalResponse extends ActionResponse implements ToXContent {
private Collection<RankEvalResult> qualityResults = new ArrayList<>();
private RankEvalResult qualityResult;
public RankEvalResponse() {
}
public RankEvalResponse(StreamInput in) throws IOException {
int size = in.readInt();
qualityResults = new ArrayList<>(size);
for (int i = 0; i < size; i++) {
qualityResults.add(new RankEvalResult(in));
}
super.readFrom(in);
this.qualityResult = new RankEvalResult(in);
}
@Override
public void writeTo(StreamOutput out) throws IOException {
super.writeTo(out);
out.writeInt(qualityResults.size());
for (RankEvalResult result : qualityResults) {
result.writeTo(out);
}
}
public void addRankEvalResult(int specId, double quality, Map<Integer, Collection<String>> unknownDocs) {
RankEvalResult result = new RankEvalResult(specId, quality, unknownDocs);
this.qualityResults.add(result);
qualityResult.writeTo(out);
}
public Collection<RankEvalResult> getRankEvalResults() {
return qualityResults;
public void setRankEvalResult(RankEvalResult result) {
this.qualityResult = result;
}
public RankEvalResult getRankEvalResult() {
return qualityResult;
}
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject("rank_eval");
builder.field("spec_id", qualityResult.getSpecId());
builder.field("quality_level", qualityResult.getQualityLevel());
builder.startArray("unknown_docs");
Map<String, Collection<String>> unknownDocs = qualityResult.getUnknownDocs();
for (String key : unknownDocs.keySet()) {
builder.startObject();
builder.field(key, unknownDocs.get(key));
builder.endObject();
}
builder.endArray();
builder.endObject();
return builder;
}
}

30
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java
View File

@ -31,22 +31,23 @@ import java.util.Map;
* For each precision at n computation the id of the search request specification used to generate search requests is returned
* for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned.
* */
// TODO do we need an extra class for this or it RankEvalResponse enough?
public class RankEvalResult implements Writeable {
/**ID of specification this result was generated for.*/
private int specId;
/**Average precision observed when issueing query intents with this spec.*/
/**ID of QA specification this result was generated for.*/
private String specId;
/**Average precision observed when issuing query intents with this specification.*/
private double qualityLevel;
/**Mapping from intent id to all documents seen for this intent that were not annotated.*/
private Map<Integer, Collection<String>> unknownDocs;
private Map<String, Collection<String>> unknownDocs;
@SuppressWarnings("unchecked")
public RankEvalResult(StreamInput in) throws IOException {
this.specId = in.readInt();
this.specId = in.readString();
this.qualityLevel = in.readDouble();
this.unknownDocs = (Map<Integer, Collection<String>>) in.readGenericValue();
this.unknownDocs = (Map<String, Collection<String>>) in.readGenericValue();
}
public RankEvalResult(int specId, double quality, Map<Integer, Collection<String>> unknownDocs) {
public RankEvalResult(String specId, double quality, Map<String, Collection<String>> unknownDocs) {
this.specId = specId;
this.qualityLevel = quality;
this.unknownDocs = unknownDocs;
@ -54,12 +55,12 @@ public class RankEvalResult implements Writeable {
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(specId);
out.writeString(specId);
out.writeDouble(qualityLevel);
out.writeGenericValue(getUnknownDocs());
}
public int getSpecId() {
public String getSpecId() {
return specId;
}
@ -67,7 +68,12 @@ public class RankEvalResult implements Writeable {
return qualityLevel;
}
public Map<Integer, Collection<String>> getUnknownDocs() {
public Map<String, Collection<String>> getUnknownDocs() {
return unknownDocs;
}
@Override
public String toString() {
return "RankEvalResult, ID :[" + specId + "], quality: " + qualityLevel + ", unknown docs: " + unknownDocs;
}
}

47
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
View File

@ -35,45 +35,54 @@ import java.util.Collection;
* */
public class RankEvalSpec implements Writeable {
/** Collection of query intents to check against including expected document ids.*/
private Collection<RatedQuery> intents = new ArrayList<>();
/** Collection of query specifications, that is e.g. search request templates to use for query translation. */
private Collection<QuerySpec> specifications = new ArrayList<>();
/** Definition of n in precision at n */
/** Definition of the quality metric, e.g. precision at N */
private RankedListQualityMetric eval;
/** a unique id for the whole QA task */
private String taskId;
public RankEvalSpec() {
// TODO think if no args ctor is okay
}
public RankEvalSpec(Collection<RatedQuery> intents, Collection<QuerySpec> specs, RankedListQualityMetric metric) {
this.intents = intents;
public RankEvalSpec(String taskId, Collection<QuerySpec> specs, RankedListQualityMetric metric) {
this.taskId = taskId;
this.specifications = specs;
this.eval = metric;
}
public RankEvalSpec(StreamInput in) throws IOException {
int intentSize = in.readInt();
intents = new ArrayList<>(intentSize);
for (int i = 0; i < intentSize; i++) {
intents.add(new RatedQuery(in));
}
int specSize = in.readInt();
specifications = new ArrayList<>(specSize);
for (int i = 0; i < specSize; i++) {
specifications.add(new QuerySpec(in));
}
eval = in.readNamedWriteable(RankedListQualityMetric.class); // TODO add to registry
taskId = in.readString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(intents.size());
for (RatedQuery query : intents) {
query.writeTo(out);
}
out.writeInt(specifications.size());
for (QuerySpec spec : specifications) {
spec.writeTo(out);
}
out.writeNamedWriteable(eval);
out.writeString(taskId);
}
public void setEval(RankedListQualityMetric eval) {
this.eval = eval;
}
public void setTaskId(String taskId) {
this.taskId = taskId;
}
public String getTaskId() {
return this.taskId;
}
/** Returns the precision at n configuration (containing level of n to consider).*/
@ -86,16 +95,6 @@ public class RankEvalSpec implements Writeable {
this.eval = config;
}
/** Returns a list of search intents to evaluate. */
public Collection<RatedQuery> getIntents() {
return intents;
}
/** Set a list of search intents to evaluate. */
public void setIntents(Collection<RatedQuery> intents) {
this.intents = intents;
}
/** Returns a list of intent to query translation specifications to evaluate. */
public Collection<QuerySpec> getSpecifications() {
return specifications;

35
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java
View File

@ -19,15 +19,23 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcherSupplier;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.NamedWriteable;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import org.elasticsearch.search.SearchHit;
import java.io.IOException;
import java.util.List;
/**
* Classes implementing this interface provide a means to compute the quality of a result list
* returned by some search.
*
* RelevancyLevel specifies the type of object determining the relevancy level of some known docid.
* */
public interface RankedListQualityMetric extends Evaluator {
public abstract class RankedListQualityMetric implements NamedWriteable {
/**
* Returns a single metric representing the ranking quality of a set of returned documents
@ -36,6 +44,27 @@ public interface RankedListQualityMetric extends Evaluator {
* @param hits the result hits as returned by some search
* @return some metric representing the quality of the result hit list wrt. to relevant doc ids.
* */
@Override
EvalQueryQuality evaluate(SearchHit[] hits, RatedQuery intent);
public abstract EvalQueryQuality evaluate(SearchHit[] hits, List<RatedDocument> ratedDocs);
public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
RankedListQualityMetric rc;
Token token = parser.nextToken();
if (token != XContentParser.Token.FIELD_NAME) {
throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name");
}
String metricName = parser.currentName();
switch (metricName) {
case PrecisionAtN.NAME:
rc = PrecisionAtN.fromXContent(parser, context);
break;
default:
throw new ParsingException(parser.getTokenLocation(), "[_na] unknown query metric name [{}]", metricName);
}
if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
// if we are at END_OBJECT, move to the next one...
parser.nextToken();
}
return rc;
}
}

86
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java Normal file
View File

@ -0,0 +1,86 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentParser.Token;
import java.io.IOException;
/**
* A document ID and its rating for the query QA use case.
* */
public class RatedDocument implements Writeable {
private final String docId;
private final int rating;
public RatedDocument(String docId, int rating) {
this.docId = docId;
this.rating = rating;
}
public RatedDocument(StreamInput in) throws IOException {
this.docId = in.readString();
this.rating = in.readVInt();
}
public String getDocID() {
return docId;
}
public int getRating() {
return rating;
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeString(docId);
out.writeVInt(rating);
}
public static RatedDocument fromXContent(XContentParser parser) throws IOException {
String id = null;
int rating = Integer.MIN_VALUE;
Token token;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (parser.currentToken().equals(Token.FIELD_NAME)) {
if (id != null) {
throw new ParsingException(parser.getTokenLocation(), "only one document id allowed, found [{}] but already got [{}]",
parser.currentName(), id);
}
id = parser.currentName();
} else if (parser.currentToken().equals(Token.VALUE_NUMBER)) {
rating = parser.intValue();
} else {
throw new ParsingException(parser.getTokenLocation(), "unexpected token [{}] while parsing rated document",
token);
}
}
if (id == null) {
throw new ParsingException(parser.getTokenLocation(), "didn't find document id");
}
return new RatedDocument(id, rating);
}
}

92
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedQuery.java
View File

@ -1,92 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.io.stream.StreamInput;
import org.elasticsearch.common.io.stream.StreamOutput;
import org.elasticsearch.common.io.stream.Writeable;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
/**
* Objects of this class represent one type of user query to qa. Each query comprises a user supplied id for easer referencing,
* a set of parameters as supplied by the end user to the search application as well as a set of rated documents (ratings e.g.
* supplied by manual result tagging or some form of automated click log based process).
* */
public class RatedQuery implements Writeable {
private final int intentId;
private final Map<String, Object> intentParameters;
private final Map<String, Integer> ratedDocuments;
public RatedQuery(
int intentId, Map<String, Object> intentParameters, Map<String, Integer> ratedDocuments) {
this.intentId = intentId;
this.intentParameters = intentParameters;
this.ratedDocuments = ratedDocuments;
}
public RatedQuery(StreamInput in) throws IOException {
this.intentId = in.readInt();
this.intentParameters = in.readMap();
int ratedDocsSize = in.readInt();
this.ratedDocuments = new HashMap<>(ratedDocsSize);
for (int i = 0; i < ratedDocsSize; i++) {
this.ratedDocuments.put(in.readString(), in.readInt());
}
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(intentId);
out.writeMap(intentParameters);
out.writeInt(ratedDocuments.size());
for(Entry<String, Integer> entry : ratedDocuments.entrySet()) {
out.writeString(entry.getKey());
out.writeInt(entry.getValue());
}
}
/** For easier referencing users are allowed to supply unique ids with each search intent they want to check for
* performance quality wise.*/
public int getIntentId() {
return intentId;
}
/**
* Returns a mapping from query parameter name to real parameter - ideally as parsed from real user logs.
* */
public Map<String, Object> getIntentParameters() {
return intentParameters;
}
/**
* Returns a set of documents and their ratings as supplied by the users.
* */
public Map<String, Integer> getRatedDocuments() {
return ratedDocuments;
}
}

88
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java
View File

@ -20,21 +20,29 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.ParsingException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.inject.Inject;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ObjectParser;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.rest.BaseRestHandler;
import org.elasticsearch.rest.RestChannel;
import org.elasticsearch.rest.RestController;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.rest.action.support.RestActions;
import org.elasticsearch.rest.action.support.RestToXContentListener;
import org.elasticsearch.search.aggregations.AggregatorParsers;
import org.elasticsearch.search.suggest.Suggesters;
import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import static org.elasticsearch.rest.RestRequest.Method.GET;
import static org.elasticsearch.rest.RestRequest.Method.POST;
@ -45,7 +53,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
* General Format:
*
*
{ "requests": [{
{
"spec_id": "human_readable_id",
"requests": [{
"id": "human_readable_id",
"request": { ... request to check ... },
"ratings": { ... mapping from doc id to rating value ... }
@ -53,12 +63,15 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
"metric": {
"... metric_name... ": {
"... metric_parameter_key ...": ...metric_parameter_value...
}}}
}
}
}
*
* Example:
*
*
{"requests": [{
{"spec_id": "huge_weight_on_location",
"requests": [{
"id": "amsterdam_query",
"request": {
"query": {
@ -78,6 +91,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
"3": 1,
"4": 1
}
}
}, {
"id": "berlin_query",
"request": {
@ -100,7 +114,9 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
}],
"metric": {
"precisionAtN": {
"size": 10}}
"size": 10
}
}
}
*
@ -135,7 +151,7 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
"failed": 0
},
"rank_eval": [{
"spec_id": "huge_weight_on_city",
"spec_id": "huge_weight_on_location",
"quality_level": 0.4,
"unknown_docs": [{
"amsterdam_query": [5, 10, 23]
@ -149,10 +165,17 @@ import static org.elasticsearch.rest.RestRequest.Method.POST;
* */
public class RestRankEvalAction extends BaseRestHandler {
private IndicesQueriesRegistry queryRegistry;
private AggregatorParsers aggregators;
private Suggesters suggesters;
@Inject
public RestRankEvalAction(Settings settings, RestController controller, IndicesQueriesRegistry queryRegistry,
AggregatorParsers aggParsers, Suggesters suggesters) {
super(settings);
this.queryRegistry = queryRegistry;
this.aggregators = aggParsers;
this.suggesters = suggesters;
controller.registerHandler(GET, "/_rank_eval", this);
controller.registerHandler(POST, "/_rank_eval", this);
controller.registerHandler(GET, "/{index}/_rank_eval", this);
@ -164,22 +187,47 @@ public class RestRankEvalAction extends BaseRestHandler {
@Override
public void handleRequest(final RestRequest request, final RestChannel channel, final NodeClient client) throws IOException {
RankEvalRequest rankEvalRequest = new RankEvalRequest();
//parseRankEvalRequest(rankEvalRequest, request, parseFieldMatcher);
//client.rankEval(rankEvalRequest, new RestStatusToXContentListener<>(channel));
}
public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, ParseFieldMatcher parseFieldMatcher)
throws IOException {
String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
BytesReference restContent = null;
if (restContent == null) {
if (RestActions.hasBodyContent(request)) {
restContent = RestActions.getRestContent(request);
BytesReference restContent = RestActions.hasBodyContent(request) ? RestActions.getRestContent(request) : null;
try (XContentParser parser = XContentFactory.xContent(restContent).createParser(restContent)) {
QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
if (restContent != null) {
parseRankEvalRequest(rankEvalRequest, request,
new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters));
}
}
if (restContent != null) {
}
client.execute(RankEvalAction.INSTANCE, rankEvalRequest, new RestToXContentListener<RankEvalResponse>(channel));
}
private static final ParseField SPECID_FIELD = new ParseField("spec_id");
private static final ParseField METRIC_FIELD = new ParseField("metric");
private static final ParseField REQUESTS_FIELD = new ParseField("requests");
private static final ObjectParser<RankEvalSpec, RankEvalContext> PARSER = new ObjectParser<>("rank_eval", RankEvalSpec::new);
static {
PARSER.declareString(RankEvalSpec::setTaskId, SPECID_FIELD);
PARSER.declareObject(RankEvalSpec::setEvaluator, (p, c) -> {
try {
return RankedListQualityMetric.fromXContent(p, c);
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex);
}
} , METRIC_FIELD);
PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> {
try {
return QuerySpec.fromXContent(p, c);
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex);
}
} , REQUESTS_FIELD);
}
public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context)
throws IOException {
List<String> indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
RankEvalSpec spec = PARSER.parse(context.parser(), context);
for (QuerySpec specification : spec.getSpecifications()) {
specification.setIndices(indices);
};
rankEvalRequest.setRankEvalSpec(spec);
}
}

56
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
View File

@ -49,18 +49,18 @@ import java.util.Map;
* Instances of this class execute a collection of search intents (read: user supplied query parameters) against a set of
* possible search requests (read: search specifications, expressed as query/search request templates) and compares the result
* against a set of annotated documents per search intent.
*
*
* If any documents are returned that haven't been annotated the document id of those is returned per search intent.
*
*
* The resulting search quality is computed in terms of precision at n and returned for each search specification for the full
* set of search intents as averaged precision at n.
* */
public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequest, RankEvalResponse> {
private SearchPhaseController searchPhaseController;
private TransportService transportService;
private SearchTransportService searchTransportService;
private ClusterService clusterService;
private ActionFilters actionFilters;
private SearchPhaseController searchPhaseController;
private TransportService transportService;
private SearchTransportService searchTransportService;
private ClusterService clusterService;
private ActionFilters actionFilters;
@Inject
public TransportRankEvalAction(Settings settings, ThreadPool threadPool, ActionFilters actionFilters,
@ -75,46 +75,36 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
this.clusterService = clusterService;
this.actionFilters = actionFilters;
// TODO this should maybe move to some registry on startup
namedWriteableRegistry.register(RankedListQualityMetric.class, PrecisionAtN.NAME, PrecisionAtN::new);
}
@Override
protected void doExecute(RankEvalRequest request, ActionListener<RankEvalResponse> listener) {
RankEvalResponse response = new RankEvalResponse();
RankEvalSpec qualityTask = request.getRankEvalSpec();
RankedListQualityMetric metric = qualityTask.getEvaluator();
for (QuerySpec spec : qualityTask.getSpecifications()) {
double qualitySum = 0;
double qualitySum = 0;
Map<String, Collection<String>> unknownDocs = new HashMap<String, Collection<String>>();
Collection<QuerySpec> specifications = qualityTask.getSpecifications();
for (QuerySpec spec : specifications) {
SearchSourceBuilder specRequest = spec.getTestRequest();
String[] indices = new String[spec.getIndices().size()];
String[] indices = new String[spec.getIndices().size()];
spec.getIndices().toArray(indices);
SearchRequest templatedRequest = new SearchRequest(indices, specRequest);
TransportSearchAction transportSearchAction = new TransportSearchAction(settings, threadPool, searchPhaseController,
transportService, searchTransportService, clusterService, actionFilters, indexNameExpressionResolver);
ActionFuture<SearchResponse> searchResponse = transportSearchAction.execute(templatedRequest);
SearchHits hits = searchResponse.actionGet().getHits();
Map<Integer, Collection<String>> unknownDocs = new HashMap<Integer, Collection<String>>();
Collection<RatedQuery> intents = qualityTask.getIntents();
for (RatedQuery intent : intents) {
TransportSearchAction transportSearchAction = new TransportSearchAction(
settings,
threadPool,
searchPhaseController,
transportService,
searchTransportService,
clusterService,
actionFilters,
indexNameExpressionResolver);
ActionFuture<SearchResponse> searchResponse = transportSearchAction.execute(templatedRequest);
SearchHits hits = searchResponse.actionGet().getHits();
EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), intent);
qualitySum += intentQuality.getQualityLevel();
unknownDocs.put(intent.getIntentId(), intentQuality.getUnknownDocs());
}
response.addRankEvalResult(spec.getSpecId(), qualitySum / intents.size(), unknownDocs);
EvalQueryQuality intentQuality = metric.evaluate(hits.getHits(), spec.getRatedDocs());
qualitySum += intentQuality.getQualityLevel();
unknownDocs.put(spec.getSpecId(), intentQuality.getUnknownDocs());
}
RankEvalResponse response = new RankEvalResponse();
RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), qualitySum / specifications.size(), unknownDocs);
response.setRankEvalResult(result);
listener.onResponse(response);
}
}

170
modules/rank-eval/src/test/java/org/elasticsearch/action/quality/PrecisionAtRequestTests.java
View File

@ -1,170 +0,0 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.quality;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.index.query.MatchQueryBuilder;
import org.elasticsearch.index.rankeval.PrecisionAtN;
import org.elasticsearch.index.rankeval.RankEvalPlugin;
import org.elasticsearch.index.rankeval.RatedQuery;
import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutionException;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0)
// NORELEASE need to fix transport client use case
public class PrecisionAtRequestTests extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return pluginList(RankEvalPlugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(RankEvalPlugin.class);
}
@Before
public void setup() {
createIndex("test");
ensureGreen();
client().prepareIndex("test", "testtype").setId("1")
.setSource("text", "berlin").get();
client().prepareIndex("test", "testtype").setId("2")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("3")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("4")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("5")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("6")
.setSource("text", "amsterdam").get();
refresh();
}
public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException {
// TODO turn into unit test - no need to execute the query here to fill hits object
MatchQueryBuilder query = new MatchQueryBuilder("text", "berlin");
SearchResponse response = client().prepareSearch().setQuery(query)
.execute().actionGet();
Map<String, Integer> relevant = new HashMap<>();
relevant.put("1", Rating.RELEVANT.ordinal());
RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant);
SearchHit[] hits = response.getHits().getHits();
assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001);
}
public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException {
// TODO turn into unit test - no need to actually execute the query here to fill the hits object
MatchQueryBuilder query = new MatchQueryBuilder("text", "amsterdam");
SearchResponse response = client().prepareSearch().setQuery(query)
.execute().actionGet();
Map<String, Integer> relevant = new HashMap<>();
relevant.put("2", Rating.RELEVANT.ordinal());
relevant.put("3", Rating.RELEVANT.ordinal());
relevant.put("4", Rating.RELEVANT.ordinal());
relevant.put("5", Rating.RELEVANT.ordinal());
relevant.put("6", Rating.IRRELEVANT.ordinal());
RatedQuery intent = new RatedQuery(0, new HashMap<>(), relevant);
SearchHit[] hits = response.getHits().getHits();
assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, intent).getQualityLevel(), 0.00001);
}
public void testPrecisionJSON() {
}
/* public void testPrecisionAction() {
// TODO turn into REST test?
Collection<RatedQuery> intents = new ArrayList<RatedQuery>();
RatedQuery intentAmsterdam = new RatedQuery(
0,
createParams("var", "amsterdam"),
createRelevant("2", "3", "4", "5"));
intents.add(intentAmsterdam);
RatedQuery intentBerlin = new RatedQuery(
1,
createParams("var", "berlin"),
createRelevant("1"));
intents.add(intentBerlin);
Collection<QuerySpec> specs = new ArrayList<QuerySpec>();
ArrayList<String> indices = new ArrayList<>();
indices.add("test");
ArrayList<String> types = new ArrayList<>();
types.add("testtype");
SearchSourceBuilder source = new SearchSourceBuilder();
QuerySpec spec = new QuerySpec(0, source, indices, types);
specs.add(spec);
RankEvalSpec task = new RankEvalSpec(intents, specs, new PrecisionAtN(10));
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(
client(),
RankEvalAction.INSTANCE,
new RankEvalRequest());
builder.setRankEvalSpec(task);
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
RankEvalResult result = response.getRankEvalResults().iterator().next();
for (Entry<Integer, Collection<String>> entry : result.getUnknownDocs().entrySet()) {
if (entry.getKey() == 0) {
assertEquals(1, entry.getValue().size());
} else {
assertEquals(0, entry.getValue().size());
}
}
}*/
private Map<String, Integer> createRelevant(String... docs) {
Map<String, Integer> relevant = new HashMap<>();
for (String doc : docs) {
relevant.put(doc, Rating.RELEVANT.ordinal());
}
return relevant;
}
private Map<String, Object> createParams(String key, String value) {
Map<String, Object> parameters = new HashMap<>();
parameters.put(key, value);
return parameters;
}
}

122
modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java Normal file
View File

@ -0,0 +1,122 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.action.quality;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.index.rankeval.PrecisionAtN;
import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
import org.elasticsearch.index.rankeval.QuerySpec;
import org.elasticsearch.index.rankeval.RankEvalAction;
import org.elasticsearch.index.rankeval.RankEvalPlugin;
import org.elasticsearch.index.rankeval.RankEvalRequest;
import org.elasticsearch.index.rankeval.RankEvalRequestBuilder;
import org.elasticsearch.index.rankeval.RankEvalResponse;
import org.elasticsearch.index.rankeval.RankEvalResult;
import org.elasticsearch.index.rankeval.RankEvalSpec;
import org.elasticsearch.index.rankeval.RatedDocument;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.test.ESIntegTestCase;
import org.junit.Before;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map.Entry;
import java.util.Set;
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, transportClientRatio = 0.0)
// NORELEASE need to fix transport client use case
public class RankEvalRequestTests extends ESIntegTestCase {
@Override
protected Collection<Class<? extends Plugin>> transportClientPlugins() {
return pluginList(RankEvalPlugin.class);
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
return pluginList(RankEvalPlugin.class);
}
@Before
public void setup() {
createIndex("test");
ensureGreen();
client().prepareIndex("test", "testtype").setId("1")
.setSource("text", "berlin").get();
client().prepareIndex("test", "testtype").setId("2")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("3")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("4")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("5")
.setSource("text", "amsterdam").get();
client().prepareIndex("test", "testtype").setId("6")
.setSource("text", "amsterdam").get();
refresh();
}
public void testPrecisionAtRequest() {
ArrayList<String> indices = new ArrayList<>();
indices.add("test");
ArrayList<String> types = new ArrayList<>();
types.add("testtype");
String specId = randomAsciiOfLength(10);
List<QuerySpec> specifications = new ArrayList<>();
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
specifications.add(new QuerySpec("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5")));
specifications.add(new QuerySpec("berlin_query", testQuery, indices, types, createRelevant("1")));
RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10));
RankEvalRequestBuilder builder = new RankEvalRequestBuilder(
client(),
RankEvalAction.INSTANCE,
new RankEvalRequest());
builder.setRankEvalSpec(task);
RankEvalResponse response = client().execute(RankEvalAction.INSTANCE, builder.request()).actionGet();
RankEvalResult result = response.getRankEvalResult();
assertEquals(specId, result.getSpecId());
assertEquals(1.0, result.getQualityLevel(), Double.MIN_VALUE);
Set<Entry<String, Collection<String>>> entrySet = result.getUnknownDocs().entrySet();
assertEquals(2, entrySet.size());
for (Entry<String, Collection<String>> entry : entrySet) {
if (entry.getKey() == "amsterdam_query") {
assertEquals(2, entry.getValue().size());
}
if (entry.getKey() == "berlin_query") {
assertEquals(5, entry.getValue().size());
}
}
}
private static List<RatedDocument> createRelevant(String... docs) {
List<RatedDocument> relevant = new ArrayList<>();
for (String doc : docs) {
relevant.add(new RatedDocument(doc, Rating.RELEVANT.ordinal()));
}
return relevant;
}
}

69
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java Normal file
View File

@ -0,0 +1,69 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.internal.InternalSearchHit;
import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
public class PrecisionAtNTests extends ESTestCase {
public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal()));
SearchHit[] hits = new InternalSearchHit[1];
hits[0] = new InternalSearchHit(0, "0", new Text("type"), Collections.emptyMap());
assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001);
}
public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException {
List<RatedDocument> rated = new ArrayList<>();
rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal()));
rated.add(new RatedDocument("1", Rating.RELEVANT.ordinal()));
rated.add(new RatedDocument("2", Rating.RELEVANT.ordinal()));
rated.add(new RatedDocument("3", Rating.RELEVANT.ordinal()));
rated.add(new RatedDocument("4", Rating.IRRELEVANT.ordinal()));
SearchHit[] hits = new InternalSearchHit[5];
for (int i = 0; i < 5; i++) {
hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap());
}
assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001);
}
public void testParseFromXContent() throws IOException {
String xContent = " {\n"
+ " \"size\": 10\n"
+ "}";
XContentParser parser = XContentFactory.xContent(xContent).createParser(xContent);
PrecisionAtN precicionAt = PrecisionAtN.fromXContent(parser, () -> ParseFieldMatcher.STRICT);
assertEquals(10, precicionAt.getN());
}
}

98
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java Normal file
View File

@ -0,0 +1,98 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ParseFieldRegistry;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.aggregations.AggregatorParsers;
import org.elasticsearch.search.suggest.Suggesters;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.List;
public class QuerySpecTests extends ESTestCase {
private static IndicesQueriesRegistry queriesRegistry;
private static SearchModule searchModule;
private static Suggesters suggesters;
private static AggregatorParsers aggsParsers;
/**
* setup for the whole base test class
*/
@BeforeClass
public static void init() throws IOException {
aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), new ParseFieldRegistry<>("aggregation_pipes"));
searchModule = new SearchModule(Settings.EMPTY, new NamedWriteableRegistry(), false);
queriesRegistry = searchModule.getQueryParserRegistry();
suggesters = searchModule.getSuggesters();
}
@AfterClass
public static void afterClass() throws Exception {
queriesRegistry = null;
searchModule = null;
suggesters = null;
aggsParsers = null;
}
public void testParseFromXContent() throws IOException {
String querySpecString = " {\n"
+ " \"id\": \"my_qa_query\",\n"
+ " \"request\": {\n"
+ " \"query\": {\n"
+ " \"bool\": {\n"
+ " \"must\": [\n"
+ " {\"match\": {\"beverage\": \"coffee\"}},\n"
+ " {\"term\": {\"browser\": {\"value\": \"safari\"}}},\n"
+ " {\"term\": {\"time_of_day\": {\"value\": \"morning\",\"boost\": 2}}},\n"
+ " {\"term\": {\"ip_location\": {\"value\": \"ams\",\"boost\": 10}}}]}\n"
+ " },\n"
+ " \"size\": 10\n"
+ " },\n"
+ " \"ratings\": [ {\"1\": 1 }, { \"2\": 0 }, { \"3\": 1 } ]\n"
+ "}";
XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString);
QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT);
RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext,
aggsParsers, suggesters);
QuerySpec specification = QuerySpec.fromXContent(parser, rankContext);
assertEquals("my_qa_query", specification.getSpecId());
assertNotNull(specification.getTestRequest());
List<RatedDocument> ratedDocs = specification.getRatedDocs();
assertEquals(3, ratedDocs.size());
assertEquals("1", ratedDocs.get(0).getDocID());
assertEquals(1, ratedDocs.get(0).getRating());
assertEquals("2", ratedDocs.get(1).getDocID());
assertEquals(0, ratedDocs.get(1).getRating());
assertEquals("3", ratedDocs.get(2).getDocID());
assertEquals(1, ratedDocs.get(2).getRating());
}
}

49
modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml
View File

@ -5,44 +5,53 @@
index:
index: foo
type: bar
id: 1
id: doc1
body: { "text": "berlin" }
- do:
index:
index: foo
type: bar
id: 2
id: doc2
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: 3
id: doc3
body: { "text": "amsterdam" }
- do:
index:
index: foo
type: bar
id: doc4
body: { "text": "something about amsterdam and berlin" }
- do:
indices.refresh: {}
- do:
rank_eval:
body:
requests: [
body: {
"spec_id" : "cities_qa_queries",
"requests" : [
{
id: "amsterdam_query",
request: { query: {match : {text : "amsterdam" }}},
ratings: { "1": 0, "2": 1, "3": 1 }
}, {
id: "berlin_query",
request: { query: { match : { text : "berlin" } }, size : 10 },
ratings: {"1": 1}
"id": "amsterdam_query",
"request": { "query": { "match" : {"text" : "amsterdam" }}},
"ratings": [{ "doc1": 0}, {"doc2": 1}, {"doc3": 1}]
},
{
"id" : "berlin_query",
"request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 },
"ratings": [{"doc1": 1}]
}
]
metric: { precisionAtN: { size: 10}}
- match: {quality_level: 1}
- gte: { took: 0 }
- is_false: task
- is_false: deleted
],
"metric" : { "precisionatn": { "size": 10}}
}
- match: {rank_eval.spec_id: "cities_qa_queries"}
- match: {rank_eval.quality_level: 1}
- match: {rank_eval.unknown_docs.0.amsterdam_query: [ "doc4"]}
- match: {rank_eval.unknown_docs.1.berlin_query: [ "doc4"]}