From 34cbc10128d07742e09cf858f2b7a75e15d7e587 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 28 Jul 2016 11:27:06 +0200 Subject: [PATCH 1/6] Add index and type information to rated doc Also add roundtrip testing of the xcontent serialisation of RatedDoc --- .../index/rankeval/QuerySpec.java | 2 +- .../index/rankeval/RatedDocument.java | 115 ++++++++++++++---- .../action/quality/RankEvalRequestTests.java | 2 +- .../index/rankeval/PrecisionAtNTests.java | 12 +- .../index/rankeval/QuerySpecTests.java | 5 +- .../test/rank_eval/10_basic.yaml | 7 +- 6 files changed, 106 insertions(+), 37 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java index 2e82fd98939..b316ab8d1d2 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/QuerySpec.java @@ -159,7 +159,7 @@ public class QuerySpec implements Writeable { } , REQUEST_FIELD); PARSER.declareObjectArray(QuerySpec::setRatedDocs, (p, c) -> { try { - return RatedDocument.fromXContent(p); + return RatedDocument.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing ratings", ex); } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index d1bd99b97c4..c76064d98f1 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -19,33 +19,83 @@ package org.elasticsearch.index.rankeval; -import org.elasticsearch.common.ParsingException; +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.common.xcontent.XContentParser.Token; import java.io.IOException; +import java.util.Objects; /** * A document ID and its rating for the query QA use case. * */ -public class RatedDocument implements Writeable { +public class RatedDocument extends ToXContentToBytes implements Writeable { - private final String docId; - private final int rating; + public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField INDEX_FIELD = new ParseField("index"); + public static final ParseField RATING_FIELD = new ParseField("rating"); - public RatedDocument(String docId, int rating) { + private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocument::new); + + static { + PARSER.declareString(RatedDocument::setIndex, INDEX_FIELD); + PARSER.declareString(RatedDocument::setType, TYPE_FIELD); + PARSER.declareString(RatedDocument::setDocId, DOC_ID_FIELD); + PARSER.declareInt(RatedDocument::setRating, RATING_FIELD); + } + + // TODO instead of docId use path to id and id itself + private String docId; + private String type; + private String index; + private int rating; + + RatedDocument() {} + + void setIndex(String index) { + this.index = index; + } + + void setType(String type) { + this.type = type; + } + + void setDocId(String docId) { + this.docId = docId; + } + + void setRating(int rating) { + this.rating = rating; + } + + public RatedDocument(String index, String type, String docId, int rating) { + this.index = index; + this.type = type; this.docId = docId; this.rating = rating; } public RatedDocument(StreamInput in) throws IOException { + this.index = in.readString(); + this.type = in.readString(); this.docId = in.readString(); this.rating = in.readVInt(); } + public String getIndex() { + return index; + } + + public String getType() { + return type; + } + public String getDocID() { return docId; } @@ -56,31 +106,44 @@ public class RatedDocument implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(type); out.writeString(docId); out.writeVInt(rating); } - public static RatedDocument fromXContent(XContentParser parser) throws IOException { - String id = null; - int rating = Integer.MIN_VALUE; - Token token; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (parser.currentToken().equals(Token.FIELD_NAME)) { - if (id != null) { - throw new ParsingException(parser.getTokenLocation(), "only one document id allowed, found [{}] but already got [{}]", - parser.currentName(), id); - } - id = parser.currentName(); - } else if (parser.currentToken().equals(Token.VALUE_NUMBER)) { - rating = parser.intValue(); - } else { - throw new ParsingException(parser.getTokenLocation(), "unexpected token [{}] while parsing rated document", - token); - } + public static RatedDocument fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.parse(parser, context); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INDEX_FIELD.getPreferredName(), index); + builder.field(TYPE_FIELD.getPreferredName(), type); + builder.field(DOC_ID_FIELD.getPreferredName(), docId); + builder.field(RATING_FIELD.getPreferredName(), rating); + builder.endObject(); + return builder; + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; } - if (id == null) { - throw new ParsingException(parser.getTokenLocation(), "didn't find document id"); + if (obj == null || getClass() != obj.getClass()) { + return false; } - return new RatedDocument(id, rating); + RatedDocument other = (RatedDocument) obj; + return Objects.equals(index, other.index) && + Objects.equals(type, other.type) && + Objects.equals(docId, other.docId) && + Objects.equals(rating, other.rating); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), index, type, docId, rating); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java index 79df86f6e56..1250b43c036 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java @@ -115,7 +115,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument(doc, Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index c123d5bbf8f..064918fe3c3 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -38,7 +38,7 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); SearchHit[] hits = new InternalSearchHit[1]; hits[0] = new InternalSearchHit(0, "0", new Text("type"), Collections.emptyMap()); assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); @@ -46,11 +46,11 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("4", Rating.IRRELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); SearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap()); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index 19f30c8b458..ecfa5684825 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -79,7 +79,10 @@ public class QuerySpecTests extends ESTestCase { + " },\n" + " \"size\": 10\n" + " },\n" - + " \"ratings\": [ {\"1\": 1 }, { \"2\": 0 }, { \"3\": 1 } ]\n" + + " \"ratings\": [ " + + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\", \"rating\" : 1 }, " + + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"2\", \"rating\" : 0 }, " + + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\", \"rating\" : 1 }]\n" + "}"; XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index d3487082b6a..0b7f8d2e813 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -40,12 +40,15 @@ { "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, - "ratings": [{ "doc1": 0}, {"doc2": 1}, {"doc3": 1}] + "ratings": [ + { "index": "foo", "type": "bar", "doc_id": "doc1", "rating": 0}, + { "index": "foo", "type": "bar", "doc_id": "doc2", "rating": 1}, + { "index": "foo", "type": "bar", "doc_id": "doc3", "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"doc1": 1}] + "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 1}] } ], "metric" : { "precisionatn": { "size": 10}} From 869e471f9f2b08bdee692eb8b33b6cac9c991e61 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Thu, 28 Jul 2016 11:42:05 +0200 Subject: [PATCH 2/6] Add missing file --- .../index/rankeval/RatedDocumentTests.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java new file mode 100644 index 00000000000..824a8c5bf27 --- /dev/null +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.common.ParseFieldMatcher; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +public class RatedDocumentTests extends ESTestCase { + + public void testXContentParsing() throws IOException { + String index = randomAsciiOfLength(10); + String type = randomAsciiOfLength(10); + String docId = randomAsciiOfLength(10); + int rating = randomInt(); + RatedDocument testItem = new RatedDocument(index, type, docId, rating); + + XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); + if (randomBoolean()) { + builder.prettyPrint(); + } + testItem.toXContent(builder, ToXContent.EMPTY_PARAMS); + XContentBuilder shuffled = shuffleXContent(builder); + XContentParser itemParser = XContentHelper.createParser(shuffled.bytes()); + itemParser.nextToken(); + + RankEvalContext context = new RankEvalContext(ParseFieldMatcher.STRICT, null, null, null); + RatedDocument parsedItem = RatedDocument.fromXContent(itemParser, context); + assertNotSame(testItem, parsedItem); + assertEquals(testItem, parsedItem); + assertEquals(testItem.hashCode(), parsedItem.hashCode()); + + } + +} From cfaa62723d9bb57f9f6e8fcba7265d3f547284dc Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Mon, 8 Aug 2016 14:09:27 +0200 Subject: [PATCH 3/6] Actually use index/type in addition to docid for comparing hits. --- .../index/rankeval/EvalQueryQuality.java | 6 +- .../index/rankeval/PrecisionAtN.java | 18 +-- .../index/rankeval/RankEvalResponse.java | 2 +- .../index/rankeval/RankEvalResult.java | 8 +- .../index/rankeval/RatedDocument.java | 72 ++++------ .../index/rankeval/RatedDocumentKey.java | 130 ++++++++++++++++++ .../rankeval/TransportRankEvalAction.java | 2 +- .../action/quality/RankEvalRequestTests.java | 7 +- .../index/rankeval/PrecisionAtNTests.java | 55 ++++++-- .../index/rankeval/QuerySpecTests.java | 12 +- .../index/rankeval/RatedDocumentTests.java | 2 +- .../test/rank_eval/10_basic.yaml | 12 +- 12 files changed, 234 insertions(+), 92 deletions(-) create mode 100644 modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java index 54edd722126..3f408b03d0f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/EvalQueryQuality.java @@ -27,14 +27,14 @@ import java.util.Collection; public class EvalQueryQuality { private double qualityLevel; - private Collection unknownDocs; + private Collection unknownDocs; - public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { + public EvalQueryQuality (double qualityLevel, Collection unknownDocs) { this.qualityLevel = qualityLevel; this.unknownDocs = unknownDocs; } - public Collection getUnknownDocs() { + public Collection getUnknownDocs() { return unknownDocs; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index 11101826ecb..292961e5c25 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -99,27 +99,27 @@ public class PrecisionAtN extends RankedListQualityMetric { @Override public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { - Collection relevantDocIds = new ArrayList<>(); - Collection irrelevantDocIds = new ArrayList<>(); + Collection relevantDocIds = new ArrayList<>(); + Collection irrelevantDocIds = new ArrayList<>(); for (RatedDocument doc : ratedDocs) { if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - relevantDocIds.add(doc.getDocID()); + relevantDocIds.add(doc.getKey()); } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - irrelevantDocIds.add(doc.getDocID()); + irrelevantDocIds.add(doc.getKey()); } } int good = 0; int bad = 0; - Collection unknownDocIds = new ArrayList(); + Collection unknownDocIds = new ArrayList(); for (int i = 0; (i < n && i < hits.length); i++) { - String id = hits[i].getId(); - if (relevantDocIds.contains(id)) { + RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); + if (relevantDocIds.contains(hitKey)) { good++; - } else if (irrelevantDocIds.contains(id)) { + } else if (irrelevantDocIds.contains(hitKey)) { bad++; } else { - unknownDocIds.add(id); + unknownDocIds.add(hitKey); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java index 0e520febdb8..4017610b9ef 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResponse.java @@ -73,7 +73,7 @@ public class RankEvalResponse extends ActionResponse implements ToXContent { builder.field("spec_id", qualityResult.getSpecId()); builder.field("quality_level", qualityResult.getQualityLevel()); builder.startArray("unknown_docs"); - Map> unknownDocs = qualityResult.getUnknownDocs(); + Map> unknownDocs = qualityResult.getUnknownDocs(); for (String key : unknownDocs.keySet()) { builder.startObject(); builder.field(key, unknownDocs.get(key)); diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java index 726b3c82aa7..64c09155769 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RankEvalResult.java @@ -39,16 +39,16 @@ public class RankEvalResult implements Writeable { /**Average precision observed when issuing query intents with this specification.*/ private double qualityLevel; /**Mapping from intent id to all documents seen for this intent that were not annotated.*/ - private Map> unknownDocs; + private Map> unknownDocs; @SuppressWarnings("unchecked") public RankEvalResult(StreamInput in) throws IOException { this.specId = in.readString(); this.qualityLevel = in.readDouble(); - this.unknownDocs = (Map>) in.readGenericValue(); + this.unknownDocs = (Map>) in.readGenericValue(); } - public RankEvalResult(String specId, double quality, Map> unknownDocs) { + public RankEvalResult(String specId, double quality, Map> unknownDocs) { this.specId = specId; this.qualityLevel = quality; this.unknownDocs = unknownDocs; @@ -69,7 +69,7 @@ public class RankEvalResult implements Writeable { return qualityLevel; } - public Map> getUnknownDocs() { + public Map> getUnknownDocs() { return unknownDocs; } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index c76064d98f1..68ca993bdac 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -21,6 +21,7 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.action.support.ToXContentToBytes; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -36,68 +37,51 @@ import java.util.Objects; * */ public class RatedDocument extends ToXContentToBytes implements Writeable { - public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); - public static final ParseField TYPE_FIELD = new ParseField("type"); - public static final ParseField INDEX_FIELD = new ParseField("index"); public static final ParseField RATING_FIELD = new ParseField("rating"); + public static final ParseField KEY_FIELD = new ParseField("key"); private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocument::new); static { - PARSER.declareString(RatedDocument::setIndex, INDEX_FIELD); - PARSER.declareString(RatedDocument::setType, TYPE_FIELD); - PARSER.declareString(RatedDocument::setDocId, DOC_ID_FIELD); + PARSER.declareObject(RatedDocument::setKey, (p, c) -> { + try { + return RatedDocumentKey.fromXContent(p, c); + } catch (IOException ex) { + throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); + } + } , KEY_FIELD); PARSER.declareInt(RatedDocument::setRating, RATING_FIELD); } - // TODO instead of docId use path to id and id itself - private String docId; - private String type; - private String index; + private RatedDocumentKey key; private int rating; RatedDocument() {} - void setIndex(String index) { - this.index = index; + void setRatedDocumentKey(RatedDocumentKey key) { + this.key = key; } - void setType(String type) { - this.type = type; + void setKey(RatedDocumentKey key) { + this.key = key; } - - void setDocId(String docId) { - this.docId = docId; - } - + void setRating(int rating) { this.rating = rating; } - public RatedDocument(String index, String type, String docId, int rating) { - this.index = index; - this.type = type; - this.docId = docId; + public RatedDocument(RatedDocumentKey key, int rating) { + this.key = key; this.rating = rating; } public RatedDocument(StreamInput in) throws IOException { - this.index = in.readString(); - this.type = in.readString(); - this.docId = in.readString(); + this.key = new RatedDocumentKey(in); this.rating = in.readVInt(); } - public String getIndex() { - return index; - } - - public String getType() { - return type; - } - - public String getDocID() { - return docId; + public RatedDocumentKey getKey() { + return this.key; } public int getRating() { @@ -106,9 +90,7 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { @Override public void writeTo(StreamOutput out) throws IOException { - out.writeString(index); - out.writeString(type); - out.writeString(docId); + this.key.writeTo(out); out.writeVInt(rating); } @@ -119,14 +101,12 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); - builder.field(INDEX_FIELD.getPreferredName(), index); - builder.field(TYPE_FIELD.getPreferredName(), type); - builder.field(DOC_ID_FIELD.getPreferredName(), docId); + builder.field(KEY_FIELD.getPreferredName(), key); builder.field(RATING_FIELD.getPreferredName(), rating); builder.endObject(); return builder; } - + @Override public final boolean equals(Object obj) { if (this == obj) { @@ -136,14 +116,12 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { return false; } RatedDocument other = (RatedDocument) obj; - return Objects.equals(index, other.index) && - Objects.equals(type, other.type) && - Objects.equals(docId, other.docId) && + return Objects.equals(key, other.key) && Objects.equals(rating, other.rating); } @Override public final int hashCode() { - return Objects.hash(getClass(), index, type, docId, rating); + return Objects.hash(getClass(), key, rating); } } diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java new file mode 100644 index 00000000000..21551921c97 --- /dev/null +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -0,0 +1,130 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.rankeval; + +import org.elasticsearch.action.support.ToXContentToBytes; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +public class RatedDocumentKey extends ToXContentToBytes implements Writeable { + public static final ParseField DOC_ID_FIELD = new ParseField("doc_id"); + public static final ParseField TYPE_FIELD = new ParseField("type"); + public static final ParseField INDEX_FIELD = new ParseField("index"); + + private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocumentKey::new); + + static { + PARSER.declareString(RatedDocumentKey::setIndex, INDEX_FIELD); + PARSER.declareString(RatedDocumentKey::setType, TYPE_FIELD); + PARSER.declareString(RatedDocumentKey::setDocId, DOC_ID_FIELD); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field(INDEX_FIELD.getPreferredName(), index); + builder.field(TYPE_FIELD.getPreferredName(), type); + builder.field(DOC_ID_FIELD.getPreferredName(), docId); + builder.endObject(); + return builder; + } + + // TODO instead of docId use path to id and id itself + private String docId; + private String type; + private String index; + + public RatedDocumentKey() {} + + + void setIndex(String index) { + this.index = index; + } + + void setType(String type) { + this.type = type; + } + + void setDocId(String docId) { + this.docId = docId; + } + + public RatedDocumentKey(String index, String type, String docId) { + this.index = index; + this.type = type; + this.docId = docId; + } + + public RatedDocumentKey(StreamInput in) throws IOException { + this.index = in.readString(); + this.type = in.readString(); + this.docId = in.readString(); + } + + public String getIndex() { + return index; + } + + public String getType() { + return type; + } + + public String getDocID() { + return docId; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(index); + out.writeString(type); + out.writeString(docId); + } + + public static RatedDocumentKey fromXContent(XContentParser parser, RankEvalContext context) throws IOException { + return PARSER.parse(parser, context); + } + + @Override + public final boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + RatedDocumentKey other = (RatedDocumentKey) obj; + return Objects.equals(index, other.index) && + Objects.equals(type, other.type) && + Objects.equals(docId, other.docId); + } + + @Override + public final int hashCode() { + return Objects.hash(getClass(), index, type, docId); + } +} diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java index ee01eb6f76c..4b1b146739f 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/TransportRankEvalAction.java @@ -85,7 +85,7 @@ public class TransportRankEvalAction extends HandledTransportAction> unknownDocs = new HashMap>(); + Map> unknownDocs = new HashMap>(); Collection specifications = qualityTask.getSpecifications(); for (QuerySpec spec : specifications) { SearchSourceBuilder specRequest = spec.getTestRequest(); diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java index 1250b43c036..5926289d3b7 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/action/quality/RankEvalRequestTests.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.rankeval.RankEvalResponse; import org.elasticsearch.index.rankeval.RankEvalResult; import org.elasticsearch.index.rankeval.RankEvalSpec; import org.elasticsearch.index.rankeval.RatedDocument; +import org.elasticsearch.index.rankeval.RatedDocumentKey; import org.elasticsearch.plugins.Plugin; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.test.ESIntegTestCase; @@ -100,9 +101,9 @@ public class RankEvalRequestTests extends ESIntegTestCase { RankEvalResult result = response.getRankEvalResult(); assertEquals(specId, result.getSpecId()); assertEquals(1.0, result.getQualityLevel(), Double.MIN_VALUE); - Set>> entrySet = result.getUnknownDocs().entrySet(); + Set>> entrySet = result.getUnknownDocs().entrySet(); assertEquals(2, entrySet.size()); - for (Entry> entry : entrySet) { + for (Entry> entry : entrySet) { if (entry.getKey() == "amsterdam_query") { assertEquals(2, entry.getValue().size()); } @@ -115,7 +116,7 @@ public class RankEvalRequestTests extends ESIntegTestCase { private static List createRelevant(String... docs) { List relevant = new ArrayList<>(); for (String doc : docs) { - relevant.add(new RatedDocument("test", "testtype", doc, Rating.RELEVANT.ordinal())); + relevant.add(new RatedDocument(new RatedDocumentKey("test", "testtype", doc), Rating.RELEVANT.ordinal())); } return relevant; } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java index 064918fe3c3..9785bb2f1fa 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/PrecisionAtNTests.java @@ -23,8 +23,9 @@ import org.elasticsearch.common.ParseFieldMatcher; import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.XContentFactory; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; @@ -38,26 +39,58 @@ public class PrecisionAtNTests extends ESTestCase { public void testPrecisionAtFiveCalculation() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - SearchHit[] hits = new InternalSearchHit[1]; - hits[0] = new InternalSearchHit(0, "0", new Text("type"), Collections.emptyMap()); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), Rating.RELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[1]; + hits[0] = new InternalSearchHit(0, "0", new Text("testtype"), Collections.emptyMap()); + hits[0].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); assertEquals(1, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); } public void testPrecisionAtFiveIgnoreOneResult() throws IOException, InterruptedException, ExecutionException { List rated = new ArrayList<>(); - rated.add(new RatedDocument("test", "testtype", "0", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "1", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "2", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "3", Rating.RELEVANT.ordinal())); - rated.add(new RatedDocument("test", "testtype", "4", Rating.IRRELEVANT.ordinal())); - SearchHit[] hits = new InternalSearchHit[5]; + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "0"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "1"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[5]; for (int i = 0; i < 5; i++) { - hits[i] = new InternalSearchHit(i, i+"", new Text("type"), Collections.emptyMap()); + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } assertEquals((double) 4 / 5, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); } + public void testPrecisionAtFiveCorrectIndex() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "0"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test_other", "testtype", "1"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + } + + public void testPrecisionAtFiveCorrectType() throws IOException, InterruptedException, ExecutionException { + List rated = new ArrayList<>(); + rated.add(new RatedDocument(new RatedDocumentKey("test", "other_type", "0"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "other_type", "1"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "2"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "3"), Rating.RELEVANT.ordinal())); + rated.add(new RatedDocument(new RatedDocumentKey("test", "testtype", "4"), Rating.IRRELEVANT.ordinal())); + InternalSearchHit[] hits = new InternalSearchHit[5]; + for (int i = 0; i < 5; i++) { + hits[i] = new InternalSearchHit(i, i+"", new Text("testtype"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); + } + assertEquals((double) 2 / 3, (new PrecisionAtN(5)).evaluate(hits, rated).getQualityLevel(), 0.00001); + } + public void testParseFromXContent() throws IOException { String xContent = " {\n" + " \"size\": 10\n" diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java index d47dfb20745..4d25fce1d91 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/QuerySpecTests.java @@ -81,9 +81,9 @@ public class QuerySpecTests extends ESTestCase { + " \"size\": 10\n" + " },\n" + " \"ratings\": [ " - + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\", \"rating\" : 1 }, " - + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"2\", \"rating\" : 0 }, " - + " {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\", \"rating\" : 1 }]\n" + + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"1\"}, \"rating\" : 1 }, " + + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"2\"}, \"rating\" : 0 }, " + + " {\"key\": {\"index\": \"test\", \"type\": \"testtype\", \"doc_id\": \"3\"}, \"rating\" : 1 }]\n" + "}"; XContentParser parser = XContentFactory.xContent(querySpecString).createParser(querySpecString); QueryParseContext queryContext = new QueryParseContext(queriesRegistry, parser, ParseFieldMatcher.STRICT); @@ -94,11 +94,11 @@ public class QuerySpecTests extends ESTestCase { assertNotNull(specification.getTestRequest()); List ratedDocs = specification.getRatedDocs(); assertEquals(3, ratedDocs.size()); - assertEquals("1", ratedDocs.get(0).getDocID()); + assertEquals("1", ratedDocs.get(0).getKey().getDocID()); assertEquals(1, ratedDocs.get(0).getRating()); - assertEquals("2", ratedDocs.get(1).getDocID()); + assertEquals("2", ratedDocs.get(1).getKey().getDocID()); assertEquals(0, ratedDocs.get(1).getRating()); - assertEquals("3", ratedDocs.get(2).getDocID()); + assertEquals("3", ratedDocs.get(2).getKey().getDocID()); assertEquals(1, ratedDocs.get(2).getRating()); } } diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java index 824a8c5bf27..ba0b1bece22 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/RatedDocumentTests.java @@ -37,7 +37,7 @@ public class RatedDocumentTests extends ESTestCase { String type = randomAsciiOfLength(10); String docId = randomAsciiOfLength(10); int rating = randomInt(); - RatedDocument testItem = new RatedDocument(index, type, docId, rating); + RatedDocument testItem = new RatedDocument(new RatedDocumentKey(index, type, docId), rating); XContentBuilder builder = XContentFactory.contentBuilder(randomFrom(XContentType.values())); if (randomBoolean()) { diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 0b7f8d2e813..848debf3403 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -41,14 +41,14 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, "ratings": [ - { "index": "foo", "type": "bar", "doc_id": "doc1", "rating": 0}, - { "index": "foo", "type": "bar", "doc_id": "doc2", "rating": 1}, - { "index": "foo", "type": "bar", "doc_id": "doc3", "rating": 1}] + {"key": { "index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 0}, + {"key": { "index": "foo", "type": "bar", "doc_id": "doc2"}, "rating": 1}, + {"key": { "index": "foo", "type": "bar", "doc_id": "doc3"}, "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, - "ratings": [{"index": "foo", "type": "bar", "doc_id": "doc1", "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc1"}, "rating": 1}] } ], "metric" : { "precisionatn": { "size": 10}} @@ -56,5 +56,5 @@ - match: {rank_eval.spec_id: "cities_qa_queries"} - match: {rank_eval.quality_level: 1} - - match: {rank_eval.unknown_docs.0.amsterdam_query: [ "doc4"]} - - match: {rank_eval.unknown_docs.1.berlin_query: [ "doc4"]} + - match: {rank_eval.unknown_docs.0.amsterdam_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} + - match: {rank_eval.unknown_docs.1.berlin_query: [ {"index": "foo", "type": "bar", "doc_id": "doc4"}]} From b2fa7c4d96fb6ba14d708101310115329c78d655 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Mon, 8 Aug 2016 14:41:58 +0200 Subject: [PATCH 4/6] Fix merge errors. --- .../index/rankeval/PrecisionAtN.java | 2 +- .../index/rankeval/ReciprocalRank.java | 12 ++++---- .../index/rankeval/ReciprocalRankTests.java | 28 +++++++++++++------ .../test/rank_eval/10_basic.yaml | 8 +++--- 4 files changed, 31 insertions(+), 19 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java index a46a3f58eab..f97ba7c7975 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/PrecisionAtN.java @@ -111,7 +111,7 @@ public class PrecisionAtN extends RankedListQualityMetric { int good = 0; int bad = 0; - Collection unknownDocIds = new ArrayList(); + Collection unknownDocIds = new ArrayList<>(); for (int i = 0; (i < n && i < hits.length); i++) { RatedDocumentKey hitKey = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); if (relevantDocIds.contains(hitKey)) { diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java index 4162896ef7d..70ec185fcbe 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/ReciprocalRank.java @@ -95,21 +95,21 @@ public class ReciprocalRank extends RankedListQualityMetric { **/ @Override public EvalQueryQuality evaluate(SearchHit[] hits, List ratedDocs) { - Set relevantDocIds = new HashSet<>(); - Set irrelevantDocIds = new HashSet<>(); + Set relevantDocIds = new HashSet<>(); + Set irrelevantDocIds = new HashSet<>(); for (RatedDocument doc : ratedDocs) { if (Rating.RELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - relevantDocIds.add(doc.getDocID()); + relevantDocIds.add(doc.getKey()); } else if (Rating.IRRELEVANT.equals(RatingMapping.mapTo(doc.getRating()))) { - irrelevantDocIds.add(doc.getDocID()); + irrelevantDocIds.add(doc.getKey()); } } - Collection unknownDocIds = new ArrayList<>(); + Collection unknownDocIds = new ArrayList<>(); int firstRelevant = -1; boolean found = false; for (int i = 0; i < hits.length; i++) { - String id = hits[i].getId(); + RatedDocumentKey id = new RatedDocumentKey(hits[i].getIndex(), hits[i].getType(), hits[i].getId()); if (relevantDocIds.contains(id)) { if (found == false && i < maxAcceptableRank) { firstRelevant = i + 1; // add one because rank is not diff --git a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java index 12dd808cff7..1abc671606c 100644 --- a/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java +++ b/modules/rank-eval/src/test/java/org/elasticsearch/index/rankeval/ReciprocalRankTests.java @@ -20,8 +20,9 @@ package org.elasticsearch.index.rankeval; import org.elasticsearch.common.text.Text; +import org.elasticsearch.index.Index; import org.elasticsearch.index.rankeval.PrecisionAtN.Rating; -import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchShardTarget; import org.elasticsearch.search.internal.InternalSearchHit; import org.elasticsearch.test.ESTestCase; @@ -42,17 +43,22 @@ public class ReciprocalRankTests extends ESTestCase { reciprocalRank.setMaxAcceptableRank(maxRank); assertEquals(maxRank, reciprocalRank.getMaxAcceptableRank()); - SearchHit[] hits = new SearchHit[10]; + InternalSearchHit[] hits = new InternalSearchHit[10]; for (int i = 0; i < 10; i++) { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); int relevantAt = 5; for (int i = 0; i < 10; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.IRRELEVANT.ordinal())); } } @@ -67,18 +73,23 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationOneRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = new SearchHit[10]; + InternalSearchHit[] hits = new InternalSearchHit[10]; for (int i = 0; i < 10; i++) { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); // mark one of the ten docs relevant int relevantAt = randomIntBetween(0, 9); for (int i = 0; i <= 20; i++) { if (i == relevantAt) { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.RELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.RELEVANT.ordinal())); } else { - ratedDocs.add(new RatedDocument(Integer.toString(i), Rating.IRRELEVANT.ordinal())); + ratedDocs.add(new RatedDocument( + new RatedDocumentKey("test", "type", Integer.toString(i)), + Rating.IRRELEVANT.ordinal())); } } @@ -97,9 +108,10 @@ public class ReciprocalRankTests extends ESTestCase { public void testEvaluationNoRelevantInResults() { ReciprocalRank reciprocalRank = new ReciprocalRank(); - SearchHit[] hits = new SearchHit[10]; + InternalSearchHit[] hits = new InternalSearchHit[10]; for (int i = 0; i < 10; i++) { hits[i] = new InternalSearchHit(i, Integer.toString(i), new Text("type"), Collections.emptyMap()); + hits[i].shard(new SearchShardTarget("testnode", new Index("test", "uuid"), 0)); } List ratedDocs = new ArrayList<>(); EvalQueryQuality evaluation = reciprocalRank.evaluate(hits, ratedDocs); diff --git a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml index 9eed83336c4..7dd41141b33 100644 --- a/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml +++ b/modules/rank-eval/src/test/resources/rest-api-spec/test/rank_eval/10_basic.yaml @@ -114,13 +114,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": : 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] } ], "metric" : { "reciprocal_rank": {} } @@ -139,13 +139,13 @@ "id": "amsterdam_query", "request": { "query": { "match" : {"text" : "amsterdam" }}}, # doc4 should be returned in third position, so reciprocal rank is 1/3 - "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": ""doc4"}, "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] }, { "id" : "berlin_query", "request": { "query": { "match" : { "text" : "berlin" } }, "size" : 10 }, # doc1 should be returned in first position, doc3 in second, so reciprocal rank is 1/2 - "ratings": [{"key": {"index": "foo", "type": "bar": "doc_id": "doc4"}, "rating": 1}] + "ratings": [{"key": {"index": "foo", "type": "bar", "doc_id": "doc4"}, "rating": 1}] } ], "metric" : { From c2bd58e13d313fc7b9ab0838726b679780c0bb01 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 9 Aug 2016 12:22:44 +0200 Subject: [PATCH 5/6] Switch to ConstructingObjectParser --- .../index/rankeval/RatedDocument.java | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java index 68ca993bdac..8e7c1aac888 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocument.java @@ -25,7 +25,7 @@ import org.elasticsearch.common.ParsingException; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -40,24 +40,23 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { public static final ParseField RATING_FIELD = new ParseField("rating"); public static final ParseField KEY_FIELD = new ParseField("key"); - private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocument::new); - + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("rated_document", + a -> new RatedDocument((RatedDocumentKey) a[0], (Integer) a[1])); + static { - PARSER.declareObject(RatedDocument::setKey, (p, c) -> { + PARSER.declareObject(ConstructingObjectParser.constructorArg(), (p, c) -> { try { return RatedDocumentKey.fromXContent(p, c); } catch (IOException ex) { throw new ParsingException(p.getTokenLocation(), "error parsing rank request", ex); } } , KEY_FIELD); - PARSER.declareInt(RatedDocument::setRating, RATING_FIELD); + PARSER.declareInt(ConstructingObjectParser.constructorArg(), RATING_FIELD); } private RatedDocumentKey key; private int rating; - RatedDocument() {} - void setRatedDocumentKey(RatedDocumentKey key) { this.key = key; } @@ -95,7 +94,7 @@ public class RatedDocument extends ToXContentToBytes implements Writeable { } public static RatedDocument fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.parse(parser, context); + return PARSER.apply(parser, context); } @Override From 0be997232a468e2a15a24e0f3d5b95514b8cd5d1 Mon Sep 17 00:00:00 2001 From: Isabel Drost-Fromm Date: Tue, 9 Aug 2016 12:49:07 +0200 Subject: [PATCH 6/6] Add ConstructingObjectParser to RatedDocumentKey --- .../index/rankeval/RatedDocumentKey.java | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java index 21551921c97..d68149a2ca0 100644 --- a/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java +++ b/modules/rank-eval/src/main/java/org/elasticsearch/index/rankeval/RatedDocumentKey.java @@ -24,7 +24,7 @@ import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; @@ -36,14 +36,15 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { public static final ParseField TYPE_FIELD = new ParseField("type"); public static final ParseField INDEX_FIELD = new ParseField("index"); - private static final ObjectParser PARSER = new ObjectParser<>("ratings", RatedDocumentKey::new); + private static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>("ratings", + a -> new RatedDocumentKey((String) a[0], (String) a[1], (String) a[2])); static { - PARSER.declareString(RatedDocumentKey::setIndex, INDEX_FIELD); - PARSER.declareString(RatedDocumentKey::setType, TYPE_FIELD); - PARSER.declareString(RatedDocumentKey::setDocId, DOC_ID_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), INDEX_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), TYPE_FIELD); + PARSER.declareString(ConstructingObjectParser.constructorArg(), DOC_ID_FIELD); } - + @Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); @@ -53,15 +54,12 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { builder.endObject(); return builder; } - + // TODO instead of docId use path to id and id itself private String docId; private String type; private String index; - public RatedDocumentKey() {} - - void setIndex(String index) { this.index = index; } @@ -106,7 +104,7 @@ public class RatedDocumentKey extends ToXContentToBytes implements Writeable { } public static RatedDocumentKey fromXContent(XContentParser parser, RankEvalContext context) throws IOException { - return PARSER.parse(parser, context); + return PARSER.apply(parser, context); } @Override