honeymoose
/
OpenSearch
mirror of https://github.com/honeymoose/OpenSearch.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix test errors, add roundtrip testing to RankEvalSpec 

This adds roundtrip testing to RankEvalSpec, fixes issues
introduced with the previous roundtrip tests, splits
xcontent generation/parsing from actually checking the resulting
objects to deal with e.g. all evaluation metrics needing
some extra treatment.

Renames QuerySpec to RatedRequest, renames newly introduced xcontent
generation helper to conform with naming conventions.

Fixes several lines that were too long, adds missing types where
needed.
This commit is contained in:
Isabel Drost-Fromm 2016-08-25 12:09:55 +02:00
parent 87367be4e7
commit f18e23eb51
19 changed files with 275 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java
View File

@ -86,8 +86,10 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric<Discount
}
/**
* @param position number of top results to check against a given set of relevant results. Must be positive. // TODO is there a way to enforce this?
* @param normalize If set to true, dcg will be normalized (ndcg) See https://en.wikipedia.org/wiki/Discounted_cumulative_gain
* @param position number of top results to check against a given set of relevant results. Must be positive.
* // TODO is there a way to enforce this?
* @param normalize If set to true, dcg will be normalized (ndcg)
* See https://en.wikipedia.org/wiki/Discounted_cumulative_gain
* @param unknownDocRating the rating for docs the user hasn't supplied an explicit rating for
* */
public DiscountedCumulativeGainAt(int position, boolean normalize, Integer unknownDocRating) {
@ -207,14 +209,15 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric<Discount
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
//builder.startObject(NAME); // TODO roundtrip xcontent only works w/o this, wtf?
builder.startObject();
builder.startObject(NAME);
builder.field(SIZE_FIELD.getPreferredName(), this.position);
builder.field(NORMALIZE_FIELD.getPreferredName(), this.normalize);
if (unknownDocRating != null) {
builder.field(UNKNOWN_DOC_RATING_FIELD.getPreferredName(), this.unknownDocRating);
}
builder.endObject();
builder.endObject();
return builder;
}

3
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java
View File

@ -166,10 +166,11 @@ public class PrecisionAtN extends RankedListQualityMetric<PrecisionAtN> {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
//builder.startObject(NAME); TODO Why does roundtripping fail with the name?
builder.startObject();
builder.startObject(NAME);
builder.field(SIZE_FIELD.getPreferredName(), this.n);
builder.endObject();
builder.endObject();
return builder;
}

67
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java
View File

@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Objects;
/**
* This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric.
@ -43,9 +44,9 @@ import java.util.Collection;
public class RankEvalSpec extends ToXContentToBytes implements Writeable {
/** Collection of query specifications, that is e.g. search request templates to use for query translation. */
private Collection<QuerySpec> specifications = new ArrayList<>();
private Collection<RatedRequest> ratedRequests = new ArrayList<>();
/** Definition of the quality metric, e.g. precision at N */
private RankedListQualityMetric eval;
private RankedListQualityMetric<?> metric;
/** a unique id for the whole QA task */
private String specId;
@ -53,34 +54,34 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable {
// TODO think if no args ctor is okay
}
public RankEvalSpec(String specId, Collection<QuerySpec> specs, RankedListQualityMetric metric) {
public RankEvalSpec(String specId, Collection<RatedRequest> specs, RankedListQualityMetric<?> metric) {
this.specId = specId;
this.specifications = specs;
this.eval = metric;
this.ratedRequests = specs;
this.metric = metric;
}
public RankEvalSpec(StreamInput in) throws IOException {
int specSize = in.readInt();
specifications = new ArrayList<>(specSize);
ratedRequests = new ArrayList<>(specSize);
for (int i = 0; i < specSize; i++) {
specifications.add(new QuerySpec(in));
ratedRequests.add(new RatedRequest(in));
}
eval = in.readNamedWriteable(RankedListQualityMetric.class);
metric = in.readNamedWriteable(RankedListQualityMetric.class);
specId = in.readString();
}
@Override
public void writeTo(StreamOutput out) throws IOException {
out.writeInt(specifications.size());
for (QuerySpec spec : specifications) {
out.writeInt(ratedRequests.size());
for (RatedRequest spec : ratedRequests) {
spec.writeTo(out);
}
out.writeNamedWriteable(eval);
out.writeNamedWriteable(metric);
out.writeString(specId);
}
public void setEval(RankedListQualityMetric eval) {
this.eval = eval;
public void setEval(RankedListQualityMetric<?> eval) {
this.metric = eval;
}
public void setTaskId(String taskId) {
@ -92,23 +93,23 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable {
}
/** Returns the precision at n configuration (containing level of n to consider).*/
public RankedListQualityMetric getEvaluator() {
return eval;
public RankedListQualityMetric<?> getEvaluator() {
return metric;
}
/** Sets the precision at n configuration (containing level of n to consider).*/
public void setEvaluator(RankedListQualityMetric config) {
this.eval = config;
public void setEvaluator(RankedListQualityMetric<?> config) {
this.metric = config;
}
/** Returns a list of intent to query translation specifications to evaluate. */
public Collection<QuerySpec> getSpecifications() {
return specifications;
public Collection<RatedRequest> getSpecifications() {
return ratedRequests;
}
/** Set the list of intent to query translation specifications to evaluate. */
public void setSpecifications(Collection<QuerySpec> specifications) {
this.specifications = specifications;
public void setSpecifications(Collection<RatedRequest> specifications) {
this.ratedRequests = specifications;
}
private static final ParseField SPECID_FIELD = new ParseField("spec_id");
@ -127,7 +128,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable {
} , METRIC_FIELD);
PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> {
try {
return QuerySpec.fromXContent(p, c);
return RatedRequest.fromXContent(p, c);
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex);
}
@ -139,11 +140,11 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable {
builder.startObject();
builder.field(SPECID_FIELD.getPreferredName(), this.specId);
builder.startArray(REQUESTS_FIELD.getPreferredName());
for (QuerySpec spec : this.specifications) {
for (RatedRequest spec : this.ratedRequests) {
spec.toXContent(builder, params);
}
builder.endArray();
builder.field(METRIC_FIELD.getPreferredName(), this.eval);
builder.field(METRIC_FIELD.getPreferredName(), this.metric);
builder.endObject();
return builder;
}
@ -152,4 +153,22 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable {
return PARSER.parse(parser, context);
}
@Override
public final boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null || getClass() != obj.getClass()) {
return false;
}
RankEvalSpec other = (RankEvalSpec) obj;
return Objects.equals(specId, other.specId) &&
Objects.equals(ratedRequests, other.ratedRequests) &&
Objects.equals(metric, other.metric);
}
@Override
public final int hashCode() {
return Objects.hash(getClass(), specId, ratedRequests, metric);
}
}

6
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java
View File

@ -39,7 +39,7 @@ import java.util.List;
*
* RelevancyLevel specifies the type of object determining the relevancy level of some known docid.
* */
public abstract class RankedListQualityMetric<T extends RankedListQualityMetric>
public abstract class RankedListQualityMetric<T extends RankedListQualityMetric<?>>
extends ToXContentToBytes
implements NamedWriteable, FromXContentBuilder<T> {
@ -52,8 +52,8 @@ public abstract class RankedListQualityMetric<T extends RankedListQualityMetric>
* */
public abstract EvalQueryQuality evaluate(SearchHit[] hits, List<RatedDocument> ratedDocs);
public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
RankedListQualityMetric rc;
public static RankedListQualityMetric<?> fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
RankedListQualityMetric<?> rc;
Token token = parser.nextToken();
if (token != XContentParser.Token.FIELD_NAME) {
throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name");

3
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java
View File

@ -43,7 +43,8 @@ public class RatedDocument extends ToXContentToBytes implements Writeable, FromX
public static final ParseField RATING_FIELD = new ParseField("rating");
public static final ParseField KEY_FIELD = new ParseField("key");
private static final ConstructingObjectParser<RatedDocument, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>("rated_document",
private static final ConstructingObjectParser<RatedDocument, ParseFieldMatcherSupplier> PARSER =
new ConstructingObjectParser<>("rated_document",
a -> new RatedDocument((RatedDocumentKey) a[0], (Integer) a[1]));
static {

6
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java
View File

@ -39,7 +39,8 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr
public static final ParseField TYPE_FIELD = new ParseField("type");
public static final ParseField INDEX_FIELD = new ParseField("index");
private static final ConstructingObjectParser<RatedDocumentKey, ParseFieldMatcherSupplier> PARSER = new ConstructingObjectParser<>("ratings",
private static final ConstructingObjectParser<RatedDocumentKey, ParseFieldMatcherSupplier> PARSER =
new ConstructingObjectParser<>("ratings",
a -> new RatedDocumentKey((String) a[0], (String) a[1], (String) a[2]));
static {
@ -117,7 +118,8 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr
});
}
public static RatedDocumentKey fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
public static RatedDocumentKey fromXContent(
XContentParser parser, ParseFieldMatcherSupplier context) throws IOException {
return PARSER.apply(parser, context);
}

22
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java → modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java
View File

@ -41,7 +41,7 @@ import java.util.Objects;
*
* The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery.
* */
public class QuerySpec extends ToXContentToBytes implements Writeable {
public class RatedRequest extends ToXContentToBytes implements Writeable {
private String specId;
private SearchSourceBuilder testRequest;
@ -50,12 +50,12 @@ public class QuerySpec extends ToXContentToBytes implements Writeable {
/** Collection of rated queries for this query QA specification.*/
private List<RatedDocument> ratedDocs = new ArrayList<>();
public QuerySpec() {
public RatedRequest() {
// ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser
// TODO decide if we can require only id as mandatory, set default values for the rest?
}
public QuerySpec(String specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types,
public RatedRequest(String specId, SearchSourceBuilder testRequest, List<String> indices, List<String> types,
List<RatedDocument> ratedDocs) {
this.specId = specId;
this.testRequest = testRequest;
@ -64,7 +64,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable {
this.ratedDocs = ratedDocs;
}
public QuerySpec(StreamInput in) throws IOException {
public RatedRequest(StreamInput in) throws IOException {
this.specId = in.readString();
testRequest = new SearchSourceBuilder(in);
int indicesSize = in.readInt();
@ -149,18 +149,18 @@ public class QuerySpec extends ToXContentToBytes implements Writeable {
private static final ParseField ID_FIELD = new ParseField("id");
private static final ParseField REQUEST_FIELD = new ParseField("request");
private static final ParseField RATINGS_FIELD = new ParseField("ratings");
private static final ObjectParser<QuerySpec, RankEvalContext> PARSER = new ObjectParser<>("requests", QuerySpec::new);
private static final ObjectParser<RatedRequest, RankEvalContext> PARSER = new ObjectParser<>("requests", RatedRequest::new);
static {
PARSER.declareString(QuerySpec::setSpecId, ID_FIELD);
PARSER.declareObject(QuerySpec::setTestRequest, (p, c) -> {
PARSER.declareString(RatedRequest::setSpecId, ID_FIELD);
PARSER.declareObject(RatedRequest::setTestRequest, (p, c) -> {
try {
return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters());
} catch (IOException ex) {
throw new ParsingException(p.getTokenLocation(), "error parsing request", ex);
}
} , REQUEST_FIELD);
PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> {
PARSER.declareObjectArray(RatedRequest::setRatedDocs, (p, c) -> {
try {
return RatedDocument.fromXContent(p, c);
} catch (IOException ex) {
@ -170,7 +170,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable {
}
/**
* Parses {@link QuerySpec} from rest representation:
* Parses {@link RatedRequest} from rest representation:
*
* Example:
* {
@ -189,7 +189,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable {
* "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ]
* }
*/
public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException {
public static RatedRequest fromXContent(XContentParser parser, RankEvalContext context) throws IOException {
return PARSER.parse(parser, context);
}
@ -215,7 +215,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable {
if (obj == null || getClass() != obj.getClass()) {
return false;
}
QuerySpec other = (QuerySpec) obj;
RatedRequest other = (RatedRequest) obj;
return Objects.equals(specId, other.specId) &&
Objects.equals(testRequest, other.testRequest) &&
Objects.equals(indices, other.indices) &&

4
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java
View File

@ -160,9 +160,9 @@ public class ReciprocalRank extends RankedListQualityMetric<ReciprocalRank> {
@Override
public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
builder.startObject();
//builder.startObject(NAME); // TODO for roundtripping to work
builder.startObject(NAME);
builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank);
//builder.endObject();
builder.endObject();
builder.endObject();
return builder;
}

2
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java
View File

@ -196,7 +196,7 @@ public class RestRankEvalAction extends BaseRestHandler {
List<String> indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
List<String> types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type")));
RankEvalSpec spec = RankEvalSpec.parse(context.parser(), context);
for (QuerySpec specification : spec.getSpecifications()) {
for (RatedRequest specification : spec.getSpecifications()) {
specification.setIndices(indices);
specification.setTypes(types);
};

8
modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java
View File

@ -64,11 +64,11 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
RankEvalSpec qualityTask = request.getRankEvalSpec();
Map<String, Collection<RatedDocumentKey>> unknownDocs = new ConcurrentHashMap<>();
Collection<QuerySpec> specifications = qualityTask.getSpecifications();
Collection<RatedRequest> specifications = qualityTask.getSpecifications();
AtomicInteger responseCounter = new AtomicInteger(specifications.size());
Map<String, EvalQueryQuality> partialResults = new ConcurrentHashMap<>(specifications.size());
for (QuerySpec querySpecification : specifications) {
for (RatedRequest querySpecification : specifications) {
final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask, querySpecification,
partialResults, unknownDocs, responseCounter);
SearchSourceBuilder specRequest = querySpecification.getTestRequest();
@ -85,13 +85,13 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
public static class RankEvalActionListener implements ActionListener<SearchResponse> {
private ActionListener<RankEvalResponse> listener;
private QuerySpec specification;
private RatedRequest specification;
private Map<String, EvalQueryQuality> partialResults;
private RankEvalSpec task;
private Map<String, Collection<RatedDocumentKey>> unknownDocs;
private AtomicInteger responseCounter;
public RankEvalActionListener(ActionListener<RankEvalResponse> listener, RankEvalSpec task, QuerySpec specification,
public RankEvalActionListener(ActionListener<RankEvalResponse> listener, RankEvalSpec task, RatedRequest specification,
Map<String, EvalQueryQuality> partialResults, Map<String, Collection<RatedDocumentKey>> unknownDocs,
AtomicInteger responseCounter) {
this.listener = listener;

17
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java
View File

@ -33,7 +33,7 @@ import java.util.Collections;
import java.util.List;
import java.util.concurrent.ExecutionException;
public class DiscountedCumulativeGainAtTests extends XContentRoundtripTestCase<DiscountedCumulativeGainAt> {
public class DiscountedCumulativeGainAtTests extends XContentRoundtripTests<DiscountedCumulativeGainAt> {
/**
* Assuming the docs are ranked in the following order:
@ -121,12 +121,21 @@ public class DiscountedCumulativeGainAtTests extends XContentRoundtripTestCase<D
assertEquals(true, dcgAt.getNormalize());
}
public void testXContentRoundtrip() throws IOException {
public static DiscountedCumulativeGainAt createTestItem() {
int position = randomIntBetween(0, 1000);
boolean normalize = randomBoolean();
Integer unknownDocRating = new Integer(randomIntBetween(0, 1000));
DiscountedCumulativeGainAt testItem = new DiscountedCumulativeGainAt(position, normalize, unknownDocRating);
roundtrip(testItem);
return new DiscountedCumulativeGainAt(position, normalize, unknownDocRating);
}
public void testXContentRoundtrip() throws IOException {
DiscountedCumulativeGainAt testItem = createTestItem();
XContentParser itemParser = roundtrip(testItem);
itemParser.nextToken();
itemParser.nextToken();
DiscountedCumulativeGainAt parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}

17
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java
View File

@ -37,7 +37,7 @@ import java.util.concurrent.ExecutionException;
import static java.util.Collections.emptyList;
public class PrecisionAtNTests extends XContentRoundtripTestCase<PrecisionAtN> {
public class PrecisionAtNTests extends XContentRoundtripTests<PrecisionAtN> {
public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException {
List<RatedDocument> rated = new ArrayList<>();
@ -111,10 +111,19 @@ public class PrecisionAtNTests extends XContentRoundtripTestCase<PrecisionAtN> {
assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE);
}
public void testXContentRoundtrip() throws IOException {
public static PrecisionAtN createTestItem() {
int position = randomIntBetween(0, 1000);
return new PrecisionAtN(position);
}
PrecisionAtN testItem = new PrecisionAtN(position);
roundtrip(testItem);
public void testXContentRoundtrip() throws IOException {
PrecisionAtN testItem = createTestItem();
XContentParser itemParser = roundtrip(testItem);
itemParser.nextToken();
itemParser.nextToken();
PrecisionAtN parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}

25
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java
View File

@ -70,13 +70,23 @@ public class QuerySpecTests extends ESTestCase {
searchRequestParsers = null;
}
public void testXContentRoundtrip() throws IOException {
public static RatedRequest createTestItem(List<String> indices, List<String> types) {
String specId = randomAsciiOfLength(50);
SearchSourceBuilder testRequest = new SearchSourceBuilder();
testRequest.size(23);
testRequest.query(new MatchAllQueryBuilder());
List<RatedDocument> ratedDocs = new ArrayList<>();
int size = randomIntBetween(0, 2);
for (int i = 0; i < size; i++) {
ratedDocs.add(RatedDocumentTests.createTestItem());
}
return new RatedRequest(specId, testRequest, indices, types, ratedDocs);
}
public void testXContentRoundtrip() throws IOException {
List<String> indices = new ArrayList<>();
int size = randomIntBetween(0, 20);
for (int i = 0; i < size; i++) {
@ -89,14 +99,7 @@ public class QuerySpecTests extends ESTestCase {
types.add(randomAsciiOfLengthBetween(0, 50));
}
List<RatedDocument> ratedDocs = new ArrayList<>();
size = randomIntBetween(0, 20);
for (int i = 0; i < size; i++) {
ratedDocs.add(RatedDocumentTests.createTestItem());
}
QuerySpec testItem = new QuerySpec(specId, testRequest, indices, types, ratedDocs);
RatedRequest testItem = createTestItem(indices, types);
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
@ -111,7 +114,7 @@ public class QuerySpecTests extends ESTestCase {
RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext,
searchRequestParsers);
QuerySpec parsedItem = QuerySpec.fromXContent(itemParser, rankContext);
RatedRequest parsedItem = RatedRequest.fromXContent(itemParser, rankContext);
parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction
parsedItem.setTypes(types); // IRL these come from URL parameters - see RestRankEvalAction
assertNotSame(testItem, parsedItem);
@ -142,7 +145,7 @@ public class QuerySpecTests extends ESTestCase {
QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT);
RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext,
searchRequestParsers);
QuerySpec specification = QuerySpec.fromXContent(parser, rankContext);
RatedRequest specification = RatedRequest.fromXContent(parser, rankContext);
assertEquals("my_qa_query", specification.getSpecId());
assertNotNull(specification.getTestRequest());
List<RatedDocument> ratedDocs = specification.getRatedDocs();

12
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java
View File

@ -72,11 +72,11 @@ public class RankEvalRequestTests extends ESIntegTestCase {
List<String> types = Arrays.asList(new String[] { "testtype" });
String specId = randomAsciiOfLength(10);
List<QuerySpec> specifications = new ArrayList<>();
List<RatedRequest> specifications = new ArrayList<>();
SearchSourceBuilder testQuery = new SearchSourceBuilder();
testQuery.query(new MatchAllQueryBuilder());
specifications.add(new QuerySpec("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5")));
specifications.add(new QuerySpec("berlin_query", testQuery, indices, types, createRelevant("1")));
specifications.add(new RatedRequest("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5")));
specifications.add(new RatedRequest("berlin_query", testQuery, indices, types, createRelevant("1")));
RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10));
@ -106,14 +106,14 @@ public class RankEvalRequestTests extends ESIntegTestCase {
List<String> types = Arrays.asList(new String[] { "testtype" });
String specId = randomAsciiOfLength(10);
List<QuerySpec> specifications = new ArrayList<>();
List<RatedRequest> specifications = new ArrayList<>();
SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder();
amsterdamQuery.query(new MatchAllQueryBuilder());
specifications.add(new QuerySpec("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5")));
specifications.add(new RatedRequest("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5")));
SearchSourceBuilder brokenQuery = new SearchSourceBuilder();
RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET");
brokenQuery.query(brokenRangeQuery);
specifications.add(new QuerySpec("broken_query", brokenQuery, indices, types, createRelevant("1")));
specifications.add(new RatedRequest("broken_query", brokenQuery, indices, types, createRelevant("1")));
RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10));

119
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java Normal file
View File

@ -0,0 +1,119 @@
/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.xcontent.ParseFieldRegistry;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentFactory;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.QueryParseContext;
import org.elasticsearch.indices.query.IndicesQueriesRegistry;
import org.elasticsearch.search.SearchModule;
import org.elasticsearch.search.SearchRequestParsers;
import org.elasticsearch.search.aggregations.AggregatorParsers;
import org.elasticsearch.search.suggest.Suggesters;
import org.elasticsearch.test.ESTestCase;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import static java.util.Collections.emptyList;
public class RankEvalSpecTests extends ESTestCase {
private static SearchModule searchModule;
private static SearchRequestParsers searchRequestParsers;
/**
* setup for the whole base test class
*/
@BeforeClass
public static void init() throws IOException {
AggregatorParsers aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"),
new ParseFieldRegistry<>("aggregation_pipes"));
searchModule = new SearchModule(Settings.EMPTY, false, emptyList());
IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry();
Suggesters suggesters = searchModule.getSuggesters();
searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters);
}
@AfterClass
public static void afterClass() throws Exception {
searchModule = null;
searchRequestParsers = null;
}
public void testRoundtripping() throws IOException {
List<String> indices = new ArrayList<>();
int size = randomIntBetween(0, 20);
for (int i = 0; i < size; i++) {
indices.add(randomAsciiOfLengthBetween(0, 50));
}
List<String> types = new ArrayList<>();
size = randomIntBetween(0, 20);
for (int i = 0; i < size; i++) {
types.add(randomAsciiOfLengthBetween(0, 50));
}
List<RatedRequest> specs = new ArrayList<>();
size = randomIntBetween(1, 2); // TODO I guess requests with no query spec should be rejected...
for (int i = 0; i < size; i++) {
specs.add(QuerySpecTests.createTestItem(indices, types));
}
String specId = randomAsciiOfLengthBetween(1, 10); // TODO we should reject zero length ids ...
@SuppressWarnings("rawtypes")
RankedListQualityMetric metric;
if (randomBoolean()) {
metric = PrecisionAtNTests.createTestItem();
} else {
metric = DiscountedCumulativeGainAtTests.createTestItem();
}
RankEvalSpec testItem = new RankEvalSpec(specId, specs, metric);
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
builder.prettyPrint();
}
testItem.toXContent(builder, ToXContent.EMPTY_PARAMS);
XContentBuilder shuffled = shuffleXContent(builder);
XContentParser itemParser = XContentHelper.createParser(shuffled.bytes());
QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT);
RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext,
searchRequestParsers);
RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext);
// IRL these come from URL parameters - see RestRankEvalAction
parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);});
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}

11
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java
View File

@ -19,9 +19,12 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
public class RatedDocumentKeyTests extends XContentRoundtripTestCase<RatedDocumentKey> {
public class RatedDocumentKeyTests extends XContentRoundtripTests<RatedDocumentKey> {
public void testXContentRoundtrip() throws IOException {
String index = randomAsciiOfLengthBetween(0, 10);
@ -29,6 +32,10 @@ public class RatedDocumentKeyTests extends XContentRoundtripTestCase<RatedDocume
String docId = randomAsciiOfLengthBetween(0, 10);
RatedDocumentKey testItem = new RatedDocumentKey(index, type, docId);
roundtrip(testItem);
XContentParser itemParser = roundtrip(testItem);
RatedDocumentKey parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}

12
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java
View File

@ -19,9 +19,12 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.xcontent.XContentParser;
import java.io.IOException;
public class RatedDocumentTests extends XContentRoundtripTestCase<RatedDocument> {
public class RatedDocumentTests extends XContentRoundtripTests<RatedDocument> {
public static RatedDocument createTestItem() {
String index = randomAsciiOfLength(10);
@ -33,6 +36,11 @@ public class RatedDocumentTests extends XContentRoundtripTestCase<RatedDocument>
}
public void testXContentParsing() throws IOException {
roundtrip(createTestItem());
RatedDocument testItem = createTestItem();
XContentParser itemParser = roundtrip(testItem);
RatedDocument parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}

12
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java
View File

@ -19,7 +19,9 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.text.Text;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.index.Index;
import org.elasticsearch.index.rankeval.PrecisionAtN.Rating;
import org.elasticsearch.search.SearchShardTarget;
@ -33,7 +35,7 @@ import java.util.Vector;
import static java.util.Collections.emptyList;
public class ReciprocalRankTests extends XContentRoundtripTestCase<ReciprocalRank> {
public class ReciprocalRankTests extends XContentRoundtripTests<ReciprocalRank> {
public void testMaxAcceptableRank() {
ReciprocalRank reciprocalRank = new ReciprocalRank();
@ -128,7 +130,13 @@ public class ReciprocalRankTests extends XContentRoundtripTestCase<ReciprocalRan
int position = randomIntBetween(0, 1000);
ReciprocalRank testItem = new ReciprocalRank(position);
roundtrip(testItem);
XContentParser itemParser = roundtrip(testItem);
itemParser.nextToken();
itemParser.nextToken();
ReciprocalRank parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
}
}

11
modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTestCase.java → modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/XContentRoundtripTests.java
View File

@ -20,7 +20,6 @@
package org.elasticsearch.index.rankeval;
import org.elasticsearch.action.support.ToXContentToBytes;
import org.elasticsearch.common.ParseFieldMatcher;
import org.elasticsearch.common.xcontent.FromXContentBuilder;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.XContentBuilder;
@ -32,9 +31,9 @@ import org.elasticsearch.test.ESTestCase;
import java.io.IOException;
public class XContentRoundtripTestCase<T extends ToXContentToBytes & FromXContentBuilder<T>> extends ESTestCase {
public class XContentRoundtripTests<T extends ToXContentToBytes & FromXContentBuilder<T>> extends ESTestCase {
public void roundtrip(T testItem) throws IOException {
public XContentParser roundtrip(T testItem) throws IOException {
XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values()));
if (randomBoolean()) {
builder.prettyPrint();
@ -42,10 +41,6 @@ public class XContentRoundtripTestCase<T extends ToXContentToBytes & FromXConten
testItem.toXContent(builder, ToXContent.EMPTY_PARAMS);
XContentBuilder shuffled = shuffleXContent(builder);
XContentParser itemParser = XContentHelper.createParser(shuffled.bytes());
itemParser.nextToken();
T parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT);
assertNotSame(testItem, parsedItem);
assertEquals(testItem, parsedItem);
assertEquals(testItem.hashCode(), parsedItem.hashCode());
return itemParser;
}
}