Merge pull request #19623 from MaineC/feature/rank-eval
Add parsing of type information
This commit is contained in:
commit
0fb7dd9054
|
@ -128,6 +128,7 @@ public class PrecisionAtN extends RankedListQualityMetric {
|
||||||
return new EvalQueryQuality(precision, unknownDocIds);
|
return new EvalQueryQuality(precision, unknownDocIds);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO add abstraction that also works for other metrics
|
||||||
public enum Rating {
|
public enum Rating {
|
||||||
IRRELEVANT, RELEVANT;
|
IRRELEVANT, RELEVANT;
|
||||||
}
|
}
|
||||||
|
|
|
@ -38,6 +38,7 @@ import java.util.Map;
|
||||||
* Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned
|
* Documents of unknown quality - i.e. those that haven't been supplied in the set of annotated documents but have been returned
|
||||||
* by the search are not taken into consideration when computing precision at n - they are ignored.
|
* by the search are not taken into consideration when computing precision at n - they are ignored.
|
||||||
*
|
*
|
||||||
|
* TODO get rid of either this or RankEvalResult
|
||||||
**/
|
**/
|
||||||
public class RankEvalResponse extends ActionResponse implements ToXContent {
|
public class RankEvalResponse extends ActionResponse implements ToXContent {
|
||||||
|
|
||||||
|
|
|
@ -32,6 +32,7 @@ import java.util.Map;
|
||||||
* for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned.
|
* for reference. In addition the averaged precision and the ids of all documents returned but not found annotated is returned.
|
||||||
* */
|
* */
|
||||||
// TODO do we need an extra class for this or it RankEvalResponse enough?
|
// TODO do we need an extra class for this or it RankEvalResponse enough?
|
||||||
|
// TODO instead of just returning averages over complete results, think of other statistics, micro avg, macro avg, partial results
|
||||||
public class RankEvalResult implements Writeable {
|
public class RankEvalResult implements Writeable {
|
||||||
/**ID of QA specification this result was generated for.*/
|
/**ID of QA specification this result was generated for.*/
|
||||||
private String specId;
|
private String specId;
|
||||||
|
|
|
@ -54,6 +54,7 @@ public abstract class RankedListQualityMetric implements NamedWriteable {
|
||||||
}
|
}
|
||||||
String metricName = parser.currentName();
|
String metricName = parser.currentName();
|
||||||
|
|
||||||
|
// TODO maybe switch to using a plugable registry later?
|
||||||
switch (metricName) {
|
switch (metricName) {
|
||||||
case PrecisionAtN.NAME:
|
case PrecisionAtN.NAME:
|
||||||
rc = PrecisionAtN.fromXContent(parser, context);
|
rc = PrecisionAtN.fromXContent(parser, context);
|
||||||
|
|
|
@ -33,6 +33,7 @@ import java.io.IOException;
|
||||||
* */
|
* */
|
||||||
public class RatedDocument implements Writeable {
|
public class RatedDocument implements Writeable {
|
||||||
|
|
||||||
|
// TODO augment with index name and type name
|
||||||
private final String docId;
|
private final String docId;
|
||||||
private final int rating;
|
private final int rating;
|
||||||
|
|
||||||
|
|
|
@ -192,6 +192,7 @@ public class RestRankEvalAction extends BaseRestHandler {
|
||||||
QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
|
QueryParseContext parseContext = new QueryParseContext(queryRegistry, parser, parseFieldMatcher);
|
||||||
if (restContent != null) {
|
if (restContent != null) {
|
||||||
parseRankEvalRequest(rankEvalRequest, request,
|
parseRankEvalRequest(rankEvalRequest, request,
|
||||||
|
// TODO can we get rid of aggregators parsers and suggesters?
|
||||||
new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters));
|
new RankEvalContext(parseFieldMatcher, parseContext, aggregators, suggesters));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -224,10 +225,13 @@ public class RestRankEvalAction extends BaseRestHandler {
|
||||||
public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context)
|
public static void parseRankEvalRequest(RankEvalRequest rankEvalRequest, RestRequest request, RankEvalContext context)
|
||||||
throws IOException {
|
throws IOException {
|
||||||
List<String> indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
|
List<String> indices = Arrays.asList(Strings.splitStringByCommaToArray(request.param("index")));
|
||||||
|
List<String> types = Arrays.asList(Strings.splitStringByCommaToArray(request.param("type")));
|
||||||
RankEvalSpec spec = PARSER.parse(context.parser(), context);
|
RankEvalSpec spec = PARSER.parse(context.parser(), context);
|
||||||
for (QuerySpec specification : spec.getSpecifications()) {
|
for (QuerySpec specification : spec.getSpecifications()) {
|
||||||
specification.setIndices(indices);
|
specification.setIndices(indices);
|
||||||
|
specification.setTypes(types);
|
||||||
};
|
};
|
||||||
|
|
||||||
rankEvalRequest.setRankEvalSpec(spec);
|
rankEvalRequest.setRankEvalSpec(spec);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -92,6 +92,9 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
|
||||||
String[] indices = new String[spec.getIndices().size()];
|
String[] indices = new String[spec.getIndices().size()];
|
||||||
spec.getIndices().toArray(indices);
|
spec.getIndices().toArray(indices);
|
||||||
SearchRequest templatedRequest = new SearchRequest(indices, specRequest);
|
SearchRequest templatedRequest = new SearchRequest(indices, specRequest);
|
||||||
|
String[] types = new String[spec.getTypes().size()];
|
||||||
|
spec.getTypes().toArray(types);
|
||||||
|
templatedRequest.types(types);
|
||||||
|
|
||||||
TransportSearchAction transportSearchAction = new TransportSearchAction(settings, threadPool, searchPhaseController,
|
TransportSearchAction transportSearchAction = new TransportSearchAction(settings, threadPool, searchPhaseController,
|
||||||
transportService, searchTransportService, clusterService, actionFilters, indexNameExpressionResolver);
|
transportService, searchTransportService, clusterService, actionFilters, indexNameExpressionResolver);
|
||||||
|
@ -103,6 +106,7 @@ public class TransportRankEvalAction extends HandledTransportAction<RankEvalRequ
|
||||||
unknownDocs.put(spec.getSpecId(), intentQuality.getUnknownDocs());
|
unknownDocs.put(spec.getSpecId(), intentQuality.getUnknownDocs());
|
||||||
}
|
}
|
||||||
RankEvalResponse response = new RankEvalResponse();
|
RankEvalResponse response = new RankEvalResponse();
|
||||||
|
// TODO move averaging to actual metric, also add other statistics
|
||||||
RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), qualitySum / specifications.size(), unknownDocs);
|
RankEvalResult result = new RankEvalResult(qualityTask.getTaskId(), qualitySum / specifications.size(), unknownDocs);
|
||||||
response.setRankEvalResult(result);
|
response.setRankEvalResult(result);
|
||||||
listener.onResponse(response);
|
listener.onResponse(response);
|
||||||
|
|
|
@ -65,6 +65,7 @@ public class QuerySpecTests extends ESTestCase {
|
||||||
aggsParsers = null;
|
aggsParsers = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO add some sort of roundtrip testing like we have now for queries?
|
||||||
public void testParseFromXContent() throws IOException {
|
public void testParseFromXContent() throws IOException {
|
||||||
String querySpecString = " {\n"
|
String querySpecString = " {\n"
|
||||||
+ " \"id\": \"my_qa_query\",\n"
|
+ " \"id\": \"my_qa_query\",\n"
|
||||||
|
|
Loading…
Reference in New Issue