From f18e23eb51b89de16a2901f393a504835ad70984 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 25 Aug 2016 12:09:55 +0200 Subject: [PATCH] Fix test errors, add roundtrip testing to RankEvalSpec This adds roundtrip testing to RankEvalSpec, fixes issues introduced with the previous roundtrip tests, splits xcontent generation/parsing from actually checking the resulting objects to deal with e.g. all evaluation metrics needing some extra treatment. Renames QuerySpec to RatedRequest, renames newly introduced xcontent generation helper to conform with naming conventions. Fixes several lines that were too long, adds missing types where needed. --- .../rankeval/DiscountedCumulativeGainAt.java | 9 +- .../index/rankeval/PrecisionAtN.java | 3 +- .../index/rankeval/RankEvalSpec.java | 67 ++++++---- .../rankeval/RankedListQualityMetric.java | 6 +- .../index/rankeval/RatedDocument.java | 3 +- .../index/rankeval/RatedDocumentKey.java | 6 +- .../{QuerySpec.java => RatedRequest.java} | 22 ++-- .../index/rankeval/ReciprocalRank.java | 4 +- .../index/rankeval/RestRankEvalAction.java | 2 +- .../rankeval/TransportRankEvalAction.java | 8 +- .../DiscountedCumulativeGainAtTests.java | 17 ++- .../index/rankeval/PrecisionAtNTests.java | 17 ++- .../index/rankeval/QuerySpecTests.java | 25 ++-- .../index/rankeval/RankEvalRequestTests.java | 12 +- .../index/rankeval/RankEvalSpecTests.java | 119 ++++++++++++++++++ .../index/rankeval/RatedDocumentKeyTests.java | 11 +- .../index/rankeval/RatedDocumentTests.java | 12 +- .../index/rankeval/ReciprocalRankTests.java | 12 +- ...tCase.java => XContentRoundtripTests.java} | 11 +- 19 files changed, 275 insertions(+), 91 deletions(-) rename modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/{QuerySpec.java => RatedRequest.java} (90%) create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java rename modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/{XContentRoundtripTestCase.java => XContentRoundtripTests.java} (77%) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java index 18e79bec3b9..22d98663738 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAt.java @@ -86,8 +86,10 @@ public class DiscountedCumulativeGainAt extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - //builder.startObject(NAME); TODO Why does roundtripping fail with the name? builder.startObject(); + builder.startObject(NAME); builder.field(SIZE_FIELD.getPreferredName(), this.n); builder.endObject(); + builder.endObject(); return builder; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java index 7f88104b4eb..0aece1d0aa1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalSpec.java @@ -32,6 +32,7 @@ import org.elasticsearch.common.xcontent.XContentParser; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.Objects; /** * This class defines a ranking evaluation task including an id, a collection of queries to evaluate and the evaluation metric. @@ -43,9 +44,9 @@ import java.util.Collection; public class RankEvalSpec extends ToXContentToBytes implements Writeable { /** Collection of query specifications, that is e.g. search request templates to use for query translation. */ - private Collection specifications = new ArrayList<>(); + private Collection ratedRequests = new ArrayList<>(); /** Definition of the quality metric, e.g. precision at N */ - private RankedListQualityMetric eval; + private RankedListQualityMetric metric; /** a unique id for the whole QA task */ private String specId; @@ -53,34 +54,34 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { // TODO think if no args ctor is okay } - public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { + public RankEvalSpec(String specId, Collection specs, RankedListQualityMetric metric) { this.specId = specId; - this.specifications = specs; - this.eval = metric; + this.ratedRequests = specs; + this.metric = metric; } public RankEvalSpec(StreamInput in) throws IOException { int specSize = in.readInt(); - specifications = new ArrayList<>(specSize); + ratedRequests = new ArrayList<>(specSize); for (int i = 0; i < specSize; i++) { - specifications.add(new QuerySpec(in)); + ratedRequests.add(new RatedRequest(in)); } - eval = in.readNamedWriteable(RankedListQualityMetric.class); + metric = in.readNamedWriteable(RankedListQualityMetric.class); specId = in.readString(); } @Override public void writeTo(StreamOutput out) throws IOException { - out.writeInt(specifications.size()); - for (QuerySpec spec : specifications) { + out.writeInt(ratedRequests.size()); + for (RatedRequest spec : ratedRequests) { spec.writeTo(out); } - out.writeNamedWriteable(eval); + out.writeNamedWriteable(metric); out.writeString(specId); } - public void setEval(RankedListQualityMetric eval) { - this.eval = eval; + public void setEval(RankedListQualityMetric eval) { + this.metric = eval; } public void setTaskId(String taskId) { @@ -92,23 +93,23 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } /** Returns the precision at n configuration (containing level of n to consider).*/ - public RankedListQualityMetric getEvaluator() { - return eval; + public RankedListQualityMetric getEvaluator() { + return metric; } /** Sets the precision at n configuration (containing level of n to consider).*/ - public void setEvaluator(RankedListQualityMetric config) { - this.eval = config; + public void setEvaluator(RankedListQualityMetric config) { + this.metric = config; } /** Returns a list of intent to query translation specifications to evaluate. */ - public Collection getSpecifications() { - return specifications; + public Collection getSpecifications() { + return ratedRequests; } /** Set the list of intent to query translation specifications to evaluate. */ - public void setSpecifications(Collection specifications) { - this.specifications = specifications; + public void setSpecifications(Collection specifications) { + this.ratedRequests = specifications; } private static final ParseField SPECID_FIELD = new ParseField("spec_id"); @@ -127,7 +128,7 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { } , METRIC_FIELD); PARSER.declareObjectArray(RankEvalSpec::setSpecifications, (p, c) -> { try { - return QuerySpec.fromXContent(p, c); + return RatedRequest.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } @@ -139,11 +140,11 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { builder.startObject(); builder.field(SPECID_FIELD.getPreferredName(), this.specId); builder.startArray(REQUESTS_FIELD.getPreferredName()); - for (QuerySpec spec : this.specifications) { + for (RatedRequest spec : this.ratedRequests) { spec.toXContent(builder, params); } builder.endArray(); - builder.field(METRIC_FIELD.getPreferredName(), this.eval); + builder.field(METRIC_FIELD.getPreferredName(), this.metric); builder.endObject(); return builder; } @@ -152,4 +153,22 @@ public class RankEvalSpec extends ToXContentToBytes implements Writeable { return PARSER.parse(parser, context); } + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RankEvalSpec other = (RankEvalSpec) obj; + return Objects.equals(specId, other.specId) && + Objects.equals(ratedRequests, other.ratedRequests) && + Objects.equals(metric, other.metric); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), specId, ratedRequests, metric); + } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java index 30793d564e3..b5ce6121ffb 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankedListQualityMetric.java @@ -39,7 +39,7 @@ import java.util.List; * * RelevancyLevel specifies the type of object determining the relevancy level of some known docid. * */ -public abstract class RankedListQualityMetric +public abstract class RankedListQualityMetric> extends ToXContentToBytes implements NamedWriteable, FromXContentBuilder { @@ -52,8 +52,8 @@ public abstract class RankedListQualityMetric * */ public abstract EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs); - public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { - RankedListQualityMetric rc; + public static RankedListQualityMetric fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + RankedListQualityMetric rc; Token token = parser.nextToken(); if (token != XContentParser.Token.FIELD_NAME) { throw new ParsingException(parser.getTokenLocation(), "[_na] missing required metric name"); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index c5faecb986e..9b2dbf3fc09 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -43,7 +43,8 @@ public class RatedDocument extends ToXContentToBytes implements Writeable, FromX public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField KEY_FIELD = new ParseField("key"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("rated_document", a -> new RatedDocument((RatedDocumentKey) a[0], (Integer) a[1])); static { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index 3efcc7e16a4..cdffcf2ac57 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -39,7 +39,8 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField INDEX_FIELD = new ParseField("index"); - private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("ratings", + private static final ConstructingObjectParser PARSER = + new ConstructingObjectParser<>("ratings", a -> new RatedDocumentKey((String) a[0], (String) a[1], (String) a[2])); static { @@ -117,7 +118,8 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable, Fr }); } - public static RatedDocumentKey fromXContent(XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { + public static RatedDocumentKey fromXContent( + XContentParser parser, ParseFieldMatcherSupplier context) throws IOException { return PARSER.apply(parser, context); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java similarity index 90% rename from modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java rename to modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java index 329839750b1..2eb88a7f0e8 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedRequest.java @@ -41,7 +41,7 @@ import java.util.Objects; * * The resulting document lists can then be compared against what was specified in the set of rated documents as part of a QAQuery. * */ -public class QuerySpec extends ToXContentToBytes implements Writeable { +public class RatedRequest extends ToXContentToBytes implements Writeable { private String specId; private SearchSourceBuilder testRequest; @@ -50,12 +50,12 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { /** Collection of rated queries for this query QA specification.*/ private List ratedDocs = new ArrayList<>(); - public QuerySpec() { + public RatedRequest() { // ctor that doesn't require all args to be present immediatly is easier to use with ObjectParser // TODO decide if we can require only id as mandatory, set default values for the rest? } - public QuerySpec(String specId, SearchSourceBuilder testRequest, List indices, List types, + public RatedRequest(String specId, SearchSourceBuilder testRequest, List indices, List types, List ratedDocs) { this.specId = specId; this.testRequest = testRequest; @@ -64,7 +64,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { this.ratedDocs = ratedDocs; } - public QuerySpec(StreamInput in) throws IOException { + public RatedRequest(StreamInput in) throws IOException { this.specId = in.readString(); testRequest = new SearchSourceBuilder(in); int indicesSize = in.readInt(); @@ -149,18 +149,18 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { private static final ParseField ID_FIELD = new ParseField("id"); private static final ParseField REQUEST_FIELD = new ParseField("request"); private static final ParseField RATINGS_FIELD = new ParseField("ratings"); - private static final ObjectParser PARSER = new ObjectParser<>("requests", QuerySpec::new); + private static final ObjectParser PARSER = new ObjectParser<>("requests", RatedRequest::new); static { - PARSER.declareString(QuerySpec::setSpecId, ID_FIELD); - PARSER.declareObject(QuerySpec::setTestRequest, (p, c) -> { + PARSER.declareString(RatedRequest::setSpecId, ID_FIELD); + PARSER.declareObject(RatedRequest::setTestRequest, (p, c) -> { try { return SearchSourceBuilder.fromXContent(c.getParseContext(), c.getAggs(), c.getSuggesters()); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing request", ex); } } , REQUEST_FIELD); - PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> { + PARSER.declareObjectArray(RatedRequest::setRatedDocs, (p, c) -> { try { return RatedDocument.fromXContent(p, c); } catch (IOException ex) { @@ -170,7 +170,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { } /** - * Parses {@link QuerySpec} from rest representation: + * Parses {@link RatedRequest} from rest representation: * * Example: * { @@ -189,7 +189,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { * "ratings": [{ "1": 1 }, { "2": 0 }, { "3": 1 } ] * } */ - public static QuerySpec fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + public static RatedRequest fromXContent(XContentParser parser, RankEvalContext context) throws IOException { return PARSER.parse(parser, context); } @@ -215,7 +215,7 @@ public class QuerySpec extends ToXContentToBytes implements Writeable { if (obj == null || getClass() != obj.getClass()) { return false; } - QuerySpec other = (QuerySpec) obj; + RatedRequest other = (RatedRequest) obj; return Objects.equals(specId, other.specId) && Objects.equals(testRequest, other.testRequest) && Objects.equals(indices, other.indices) && diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index a40ecc1012e..5e9d4874e4d 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -160,9 +160,9 @@ public class ReciprocalRank extends RankedListQualityMetric { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - //builder.startObject(NAME); // TODO for roundtripping to work + builder.startObject(NAME); builder.field(MAX_RANK_FIELD.getPreferredName(), this.maxAcceptableRank); - //builder.endObject(); + builder.endObject(); builder.endObject(); return builder; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java index 8acc7b34a51..e3e5941c969 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RestRankEvalAction.java @@ -196,7 +196,7 @@ public class RestRankEvalAction extends BaseRestHandler { List indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index"))); List types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type"))); RankEvalSpec spec = RankEvalSpec.parse(context.parser(), context); - for (QuerySpec specification : spec.getSpecifications()) { + for (RatedRequest specification : spec.getSpecifications()) { specification.setIndices(indices); specification.setTypes(types); }; diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index 3e4a8d0ea19..79332a22dc7 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -64,11 +64,11 @@ public class TransportRankEvalAction extends HandledTransportAction> unknownDocs = new ConcurrentHashMap<>(); - Collection specifications = qualityTask.getSpecifications(); + Collection specifications = qualityTask.getSpecifications(); AtomicInteger responseCounter = new AtomicInteger(specifications.size()); Map partialResults = new ConcurrentHashMap<>(specifications.size()); - for (QuerySpec querySpecification : specifications) { + for (RatedRequest querySpecification : specifications) { final RankEvalActionListener searchListener = new RankEvalActionListener(listener, qualityTask, querySpecification, partialResults, unknownDocs, responseCounter); SearchSourceBuilder specRequest = querySpecification.getTestRequest(); @@ -85,13 +85,13 @@ public class TransportRankEvalAction extends HandledTransportAction { private ActionListener listener; - private QuerySpec specification; + private RatedRequest specification; private Map partialResults; private RankEvalSpec task; private Map> unknownDocs; private AtomicInteger responseCounter; - public RankEvalActionListener(ActionListener listener, RankEvalSpec task, QuerySpec specification, + public RankEvalActionListener(ActionListener listener, RankEvalSpec task, RatedRequest specification, Map partialResults, Map> unknownDocs, AtomicInteger responseCounter) { this.listener = listener; diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java index 68207470382..e92f147acea 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/DiscountedCumulativeGainAtTests.java @@ -33,7 +33,7 @@ import java.util.Collections; import java.util.List; import java.util.concurrent.ExecutionException; -public class DiscountedCumulativeGainAtTests extends XContentRoundtripTestCase { +public class DiscountedCumulativeGainAtTests extends XContentRoundtripTests { /** * Assuming the docs are ranked in the following order: @@ -121,12 +121,21 @@ public class DiscountedCumulativeGainAtTests extends XContentRoundtripTestCase { +public class PrecisionAtNTests extends XContentRoundtripTests { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); @@ -111,10 +111,19 @@ public class PrecisionAtNTests extends XContentRoundtripTestCase { assertEquals(0.3, metric.combine(partialResults), Double.MIN_VALUE); } - public void testXContentRoundtrip() throws IOException { + public static PrecisionAtN createTestItem() { int position = randomIntBetween(0, 1000); + return new PrecisionAtN(position); + } - PrecisionAtN testItem = new PrecisionAtN(position); - roundtrip(testItem); + public void testXContentRoundtrip() throws IOException { + PrecisionAtN testItem = createTestItem(); + XContentParser itemParser = roundtrip(testItem); + itemParser.nextToken(); + itemParser.nextToken(); + PrecisionAtN parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index deec3f73b1a..887abf1a652 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -70,13 +70,23 @@ public class QuerySpecTests extends ESTestCase { searchRequestParsers = null; } - public void testXContentRoundtrip() throws IOException { + public static RatedRequest createTestItem(List indices, List types) { String specId = randomAsciiOfLength(50); SearchSourceBuilder testRequest = new SearchSourceBuilder(); testRequest.size(23); testRequest.query(new MatchAllQueryBuilder()); + List ratedDocs = new ArrayList<>(); + int size = randomIntBetween(0, 2); + for (int i = 0; i < size; i++) { + ratedDocs.add(RatedDocumentTests.createTestItem()); + } + + return new RatedRequest(specId, testRequest, indices, types, ratedDocs); + } + + public void testXContentRoundtrip() throws IOException { List indices = new ArrayList<>(); int size = randomIntBetween(0, 20); for (int i = 0; i < size; i++) { @@ -89,15 +99,8 @@ public class QuerySpecTests extends ESTestCase { types.add(randomAsciiOfLengthBetween(0, 50)); } - List ratedDocs = new ArrayList<>(); - size = randomIntBetween(0, 20); - for (int i = 0; i < size; i++) { - ratedDocs.add(RatedDocumentTests.createTestItem()); - } - + RatedRequest testItem = createTestItem(indices, types); - QuerySpec testItem = new QuerySpec(specId, testRequest, indices, types, ratedDocs); - XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); @@ -111,7 +114,7 @@ public class QuerySpecTests extends ESTestCase { RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, searchRequestParsers); - QuerySpec parsedItem = QuerySpec.fromXContent(itemParser, rankContext); + RatedRequest parsedItem = RatedRequest.fromXContent(itemParser, rankContext); parsedItem.setIndices(indices); // IRL these come from URL parameters - see RestRankEvalAction parsedItem.setTypes(types); // IRL these come from URL parameters - see RestRankEvalAction assertNotSame(testItem, parsedItem); @@ -142,7 +145,7 @@ public class QuerySpecTests extends ESTestCase { QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, parser, ParseFieldMatcher.STRICT); RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, searchRequestParsers); - QuerySpec specification = QuerySpec.fromXContent(parser, rankContext); + RatedRequest specification = RatedRequest.fromXContent(parser, rankContext); assertEquals("my_qa_query", specification.getSpecId()); assertNotNull(specification.getTestRequest()); List ratedDocs = specification.getRatedDocs(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java index c03010c73b1..bae486ef0e3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalRequestTests.java @@ -72,11 +72,11 @@ public class RankEvalRequestTests extends ESIntegTestCase { List types = Arrays.asList(new String[] { "testtype" }); String specId = randomAsciiOfLength(10); - List specifications = new ArrayList<>(); + List specifications = new ArrayList<>(); SearchSourceBuilder testQuery = new SearchSourceBuilder(); testQuery.query(new MatchAllQueryBuilder()); - specifications.add(new QuerySpec("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); - specifications.add(new QuerySpec("berlin_query", testQuery, indices, types, createRelevant("1"))); + specifications.add(new RatedRequest("amsterdam_query", testQuery, indices, types, createRelevant("2", "3", "4", "5"))); + specifications.add(new RatedRequest("berlin_query", testQuery, indices, types, createRelevant("1"))); RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); @@ -106,14 +106,14 @@ public class RankEvalRequestTests extends ESIntegTestCase { List types = Arrays.asList(new String[] { "testtype" }); String specId = randomAsciiOfLength(10); - List specifications = new ArrayList<>(); + List specifications = new ArrayList<>(); SearchSourceBuilder amsterdamQuery = new SearchSourceBuilder(); amsterdamQuery.query(new MatchAllQueryBuilder()); - specifications.add(new QuerySpec("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5"))); + specifications.add(new RatedRequest("amsterdam_query", amsterdamQuery, indices, types, createRelevant("2", "3", "4", "5"))); SearchSourceBuilder brokenQuery = new SearchSourceBuilder(); RangeQueryBuilder brokenRangeQuery = new RangeQueryBuilder("text").timeZone("CET"); brokenQuery.query(brokenRangeQuery); - specifications.add(new QuerySpec("broken_query", brokenQuery, indices, types, createRelevant("1"))); + specifications.add(new RatedRequest("broken_query", brokenQuery, indices, types, createRelevant("1"))); RankEvalSpec task = new RankEvalSpec(specId, specifications, new PrecisionAtN(10)); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java new file mode 100644 index 00000000000..3fac5cb3868 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RankEvalSpecTests.java @@ -0,0 +1,119 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.ParseFieldRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.index.query.QueryParseContext; +import org.elasticsearch.indices.query.IndicesQueriesRegistry; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.SearchRequestParsers; +import org.elasticsearch.search.aggregations.AggregatorParsers; +import org.elasticsearch.search.suggest.Suggesters; +import org.elasticsearch.test.ESTestCase; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import static java.util.Collections.emptyList; + +public class RankEvalSpecTests extends ESTestCase { + private static SearchModule searchModule; + private static SearchRequestParsers searchRequestParsers; + + /** + * setup for the whole base test class + */ + @BeforeClass + public static void init() throws IOException { + AggregatorParsers aggsParsers = new AggregatorParsers(new ParseFieldRegistry<>("aggregation"), + new ParseFieldRegistry<>("aggregation_pipes")); + searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); + IndicesQueriesRegistry queriesRegistry = searchModule.getQueryParserRegistry(); + Suggesters suggesters = searchModule.getSuggesters(); + searchRequestParsers = new SearchRequestParsers(queriesRegistry, aggsParsers, suggesters); + } + + @AfterClass + public static void afterClass() throws Exception { + searchModule = null; + searchRequestParsers = null; + } + + public void testRoundtripping() throws IOException { + List indices = new ArrayList<>(); + int size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + indices.add(randomAsciiOfLengthBetween(0, 50)); + } + + List types = new ArrayList<>(); + size = randomIntBetween(0, 20); + for (int i = 0; i < size; i++) { + types.add(randomAsciiOfLengthBetween(0, 50)); + } + List specs = new ArrayList<>(); + size = randomIntBetween(1, 2); // TODO I guess requests with no query spec should be rejected... + for (int i = 0; i < size; i++) { + specs.add(QuerySpecTests.createTestItem(indices, types)); + } + + String specId = randomAsciiOfLengthBetween(1, 10); // TODO we should reject zero length ids ... + @SuppressWarnings("rawtypes") + RankedListQualityMetric metric; + if (randomBoolean()) { + metric = PrecisionAtNTests.createTestItem(); + } else { + metric = DiscountedCumulativeGainAtTests.createTestItem(); + } + + RankEvalSpec testItem = new RankEvalSpec(specId, specs, metric); + + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + + QueryParseContext queryContext = new QueryParseContext(searchRequestParsers.queryParsers, itemParser, ParseFieldMatcher.STRICT); + RankEvalContext rankContext = new RankEvalContext(ParseFieldMatcher.STRICT, queryContext, + searchRequestParsers); + + RankEvalSpec parsedItem = RankEvalSpec.parse(itemParser, rankContext); + // IRL these come from URL parameters - see RestRankEvalAction + parsedItem.getSpecifications().stream().forEach(e -> {e.setIndices(indices); e.setTypes(types);}); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + } + +} diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java index 1ec1bbbcc06..7111a0064a4 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentKeyTests.java @@ -19,9 +19,12 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.XContentParser; + import java.io.IOException; -public class RatedDocumentKeyTests extends XContentRoundtripTestCase { +public class RatedDocumentKeyTests extends XContentRoundtripTests { public void testXContentRoundtrip() throws IOException { String index = randomAsciiOfLengthBetween(0, 10); @@ -29,6 +32,10 @@ public class RatedDocumentKeyTests extends XContentRoundtripTestCase { +public class RatedDocumentTests extends XContentRoundtripTests { public static RatedDocument createTestItem() { String index = randomAsciiOfLength(10); @@ -33,6 +36,11 @@ public class RatedDocumentTests extends XContentRoundtripTestCase } public void testXContentParsing() throws IOException { - roundtrip(createTestItem()); + RatedDocument testItem = createTestItem(); + XContentParser itemParser = roundtrip(testItem); + RatedDocument parsedItem = testItem.fromXContent(itemParser, ParseFieldMatcher.STRICT); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index b25faba52d4..56443b85c11 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -19,7 +19,9 @@ package org.elasticsearch.index.rankeval; +import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; +import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; import org.elasticsearch.search.SearchShardTarget; @@ -33,7 +35,7 @@ import java.util.Vector; import static java.util.Collections.emptyList; -public class ReciprocalRankTests extends XContentRoundtripTestCase { +public class ReciprocalRankTests extends XContentRoundtripTests { public void testMaxAcceptableRank() { ReciprocalRank reciprocalRank = new ReciprocalRank(); @@ -128,7 +130,13 @@ public class ReciprocalRankTests extends XContentRoundtripTestCase> extends ESTestCase { +public class XContentRoundtripTests> extends ESTestCase { - public void roundtrip(T testItem) throws IOException { + public XContentParser roundtrip(T testItem) throws IOException { XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { builder.prettyPrint(); @@ -42,10 +41,6 @@ public class XContentRoundtripTestCase